From 45a9d5188dd4f74d5d34bd903a0dcb4b9093a73a Mon Sep 17 00:00:00 2001 From: Andrea Amorosi Date: Sat, 3 Jun 2023 23:39:32 +0000 Subject: [PATCH 1/7] docs: base docs --- .gitignore | 1 + docs/snippets/idempotency/disablingUtility.ts | 28 + .../idempotency/dynamoDBCompositePK.ts | 20 + .../idempotency/idempotentLambdaHandler.ts | 44 + docs/snippets/idempotency/idempotentMethod.ts | 4 + docs/snippets/idempotency/jestMockDynamoDB.ts | 40 + docs/snippets/idempotency/localDynamoDB.ts | 23 + .../idempotency/requiredIdempotencyKey.ts | 20 + .../idempotency/testingYourCodeFunction.ts | 23 + docs/snippets/idempotency/types.ts | 15 + docs/snippets/package.json | 2 +- docs/snippets/tsconfig.json | 6 + docs/utilities/idempotency.md | 932 ++++++++++++++++++ 13 files changed, 1157 insertions(+), 1 deletion(-) create mode 100644 docs/snippets/idempotency/disablingUtility.ts create mode 100644 docs/snippets/idempotency/dynamoDBCompositePK.ts create mode 100644 docs/snippets/idempotency/idempotentLambdaHandler.ts create mode 100644 docs/snippets/idempotency/idempotentMethod.ts create mode 100644 docs/snippets/idempotency/jestMockDynamoDB.ts create mode 100644 docs/snippets/idempotency/localDynamoDB.ts create mode 100644 docs/snippets/idempotency/requiredIdempotencyKey.ts create mode 100644 docs/snippets/idempotency/testingYourCodeFunction.ts create mode 100644 docs/snippets/idempotency/types.ts create mode 100644 docs/utilities/idempotency.md diff --git a/.gitignore b/.gitignore index 489c489c75..eab53c4c25 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ coverage # Python virtual environments (for running mkdocs locally) venv +.venv # Static documentation site generated by Mkdocs site diff --git a/docs/snippets/idempotency/disablingUtility.ts b/docs/snippets/idempotency/disablingUtility.ts new file mode 100644 index 0000000000..a3eb34b60b --- /dev/null +++ b/docs/snippets/idempotency/disablingUtility.ts @@ -0,0 +1,28 @@ +import { handler } from './testingYourCodeFunction'; + +describe('Function tests', () => { + const ENVIRONMENT_VARIABLES = process.env; + + beforeEach(() => { + process.env = { ...ENVIRONMENT_VARIABLES }; + }); + + afterAll(() => { + process.env = ENVIRONMENT_VARIABLES; + }); + + test('it returns the correct response', async () => { + // Prepare + process.env.POWERTOOLS_IDEMPOTENCY_DISABLED = 'true'; + + // Act + const result = await handler({}, {}); + + // Assess + expect(result).toStrictEqual({ + paymentId: 12345, + message: 'success', + statusCode: 200, + }); + }); +}); diff --git a/docs/snippets/idempotency/dynamoDBCompositePK.ts b/docs/snippets/idempotency/dynamoDBCompositePK.ts new file mode 100644 index 0000000000..20774a78a2 --- /dev/null +++ b/docs/snippets/idempotency/dynamoDBCompositePK.ts @@ -0,0 +1,20 @@ +import { makeHandlerIdempotent } from '@aws-lambda-powertools/idempotency/middleware'; +import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; +import middy from '@middy/core'; +import type { Request, Response } from './types'; + +const persistenceStore = new DynamoDBPersistenceLayer({ + tableName: 'IdempotencyTable', + sortKeyAttr: 'sort_key', +}); + +export const handler = middy( + async (event: Request, _context: unknown): Promise => ({ + message: 'success', + id: event.email, + }) +).use( + makeHandlerIdempotent({ + persistenceStore, + }) +); diff --git a/docs/snippets/idempotency/idempotentLambdaHandler.ts b/docs/snippets/idempotency/idempotentLambdaHandler.ts new file mode 100644 index 0000000000..ba20d4839c --- /dev/null +++ b/docs/snippets/idempotency/idempotentLambdaHandler.ts @@ -0,0 +1,44 @@ +import { idempotentLambdaHandler } from '@aws-lambda-powertools/idempotency'; +import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; +import { LambdaInterface } from '@aws-lambda-powertools/commons'; +import type { Request, Response, SubscriptionResult } from './types'; + +const persistenceStore = new DynamoDBPersistenceLayer({ + tableName: 'IdempotencyTable', +}); + +class Lambda implements LambdaInterface { + // Decorate your handler class method + @idempotentLambdaHandler({ + persistenceStore, + }) + public async handler(event: Request, _context: unknown): Promise { + const payment = await this.#createSubscriptionPayment( + event.user, + event.productId + ); + + await this.#sendNotification(event.email); + + return { + paymentId: payment.id, + statusCode: 200, + }; + } + + async #createSubscriptionPayment( + _user: string, + _product: string + ): Promise { + return { + id: 'foo', + }; + } + + async #sendNotification(_email: string): Promise { + // ... + } +} + +const myFunction = new Lambda(); +export const handler = myFunction.handler.bind(myFunction); // (1) diff --git a/docs/snippets/idempotency/idempotentMethod.ts b/docs/snippets/idempotency/idempotentMethod.ts new file mode 100644 index 0000000000..7699ba4048 --- /dev/null +++ b/docs/snippets/idempotency/idempotentMethod.ts @@ -0,0 +1,4 @@ +import { idempotentLambdaHandler } from '@aws-lambda-powertools/idempotency'; +import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; +import { LambdaInterface } from '@aws-lambda-powertools/commons'; +import type { Request, Response, SubscriptionResult } from './types'; \ No newline at end of file diff --git a/docs/snippets/idempotency/jestMockDynamoDB.ts b/docs/snippets/idempotency/jestMockDynamoDB.ts new file mode 100644 index 0000000000..967ef98cba --- /dev/null +++ b/docs/snippets/idempotency/jestMockDynamoDB.ts @@ -0,0 +1,40 @@ +import { handler } from './testingYourCodeFunction'; +import { + DynamoDBClient, + GetItemCommand, + ResourceNotFoundException, +} from '@aws-sdk/client-dynamodb'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest'; + +describe('Function tests', () => { + const client = mockClient(DynamoDBClient); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterEach(() => { + client.reset(); + }); + + test('it returns the correct response', async () => { + // Prepare + client.on(GetItemCommand).rejectsOnce( + new ResourceNotFoundException({ + $metadata: { + httpStatusCode: 404, + }, + message: 'Unable to find table', + }) + ); + + // TODO: test this test + + // Act + const result = await handler({}, {}); + + // Assess + expect(result).toStrictEqual({ message: 'Unable to find table' }); + }); +}); diff --git a/docs/snippets/idempotency/localDynamoDB.ts b/docs/snippets/idempotency/localDynamoDB.ts new file mode 100644 index 0000000000..29e28c076f --- /dev/null +++ b/docs/snippets/idempotency/localDynamoDB.ts @@ -0,0 +1,23 @@ +import { handler } from './testingYourCodeFunction'; +import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; + +describe('Function tests', () => { + test('it returns the correct response', async () => { + // Prepare + const ddbClient = new DynamoDBClient({ + endpoint: 'http://localhost:8000', + }); + + // TODO: patch the ddb client + + // Act + const result = await handler({}, {}); + + // Assess + expect(result).toStrictEqual({ + paymentId: 12345, + message: 'success', + statusCode: 200, + }); + }); +}); diff --git a/docs/snippets/idempotency/requiredIdempotencyKey.ts b/docs/snippets/idempotency/requiredIdempotencyKey.ts new file mode 100644 index 0000000000..1ad36edcc6 --- /dev/null +++ b/docs/snippets/idempotency/requiredIdempotencyKey.ts @@ -0,0 +1,20 @@ +import { + IdempotencyConfig, + makeFunctionIdempotent, +} from '@aws-lambda-powertools/idempotency'; +import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; + +const persistenceStore = new DynamoDBPersistenceLayer({ + tableName: 'IdempotencyTable', +}); + +// Requires "user"."uid" and "orderId" to be present +const config = new IdempotencyConfig({ + eventKeyJmesPath: '[user.uid, orderId]', + throwOnNoIdempotencyKey: true, +}); + +export const handler = makeFunctionIdempotent((_event: unknown) => ({}), { + persistenceStore, + config, +}); diff --git a/docs/snippets/idempotency/testingYourCodeFunction.ts b/docs/snippets/idempotency/testingYourCodeFunction.ts new file mode 100644 index 0000000000..c8449541a2 --- /dev/null +++ b/docs/snippets/idempotency/testingYourCodeFunction.ts @@ -0,0 +1,23 @@ +import { makeFunctionIdempotent } from '@aws-lambda-powertools/idempotency'; +import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; +import type { Response } from './types'; + +const persistenceStore = new DynamoDBPersistenceLayer({ + tableName: 'IdempotencyTable', +}); + +const myLambdaHandler = async ( + _event: unknown, + _context: unknown +): Promise => { + // expensive operation + return { + paymentId: 12345, + message: 'success', + statusCode: 200, + }; +}; + +export const handler = makeFunctionIdempotent(myLambdaHandler, { + persistenceStore, +}); diff --git a/docs/snippets/idempotency/types.ts b/docs/snippets/idempotency/types.ts new file mode 100644 index 0000000000..2f2997eef9 --- /dev/null +++ b/docs/snippets/idempotency/types.ts @@ -0,0 +1,15 @@ +type Request = { + user: string; + email: string; + productId: string; +}; + +type Response = { + [key: string]: unknown; +}; + +type SubscriptionResult = { + id: string; +}; + +export { Request, Response, SubscriptionResult }; diff --git a/docs/snippets/package.json b/docs/snippets/package.json index 0ce221ad60..5e067b21e3 100644 --- a/docs/snippets/package.json +++ b/docs/snippets/package.json @@ -38,4 +38,4 @@ "axios": "^1.2.4", "hashi-vault-js": "^0.4.13" } -} +} \ No newline at end of file diff --git a/docs/snippets/tsconfig.json b/docs/snippets/tsconfig.json index 6f72111cb7..6a2d536968 100644 --- a/docs/snippets/tsconfig.json +++ b/docs/snippets/tsconfig.json @@ -29,6 +29,12 @@ "@aws-lambda-powertools/parameters/dynamodb": [ "../../packages/parameters/lib/dynamodb" ], + "@aws-lambda-powertools/idempotency/dynamodb": [ + "../../packages/idempotency/lib/persistence/DynamoDBPersistenceLayer" + ], + "@aws-lambda-powertools/idempotency/persistence": [ + "../../packages/idempotency/lib/persistence" + ] }, }, "exclude": [ diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md new file mode 100644 index 0000000000..b2785de601 --- /dev/null +++ b/docs/utilities/idempotency.md @@ -0,0 +1,932 @@ +--- +title: Idempotency +description: Utility +--- + +!!! warning + This page refers to an **unreleased and upcoming utility**. Please refer to this [GitHub milestone](https://github.com/awslabs/aws-lambda-powertools-typescript/milestone/7) for the latest updates. + +The idempotency utility provides a simple solution to convert your Lambda functions into idempotent operations which are safe to retry. + +## Terminology + +The property of idempotency means that an operation does not cause additional side effects if it is called more than once with the same input parameters. + +**Idempotent operations will return the same result when they are called multiple times with the same parameters**. This makes idempotent operations safe to retry. + +**Idempotency key** is a hash representation of either the entire event or a specific configured subset of the event, and invocation results are **JSON serialized** and stored in your persistence storage layer. + +**Idempotency record** is the data representation of an idempotent request saved in your preferred storage layer. We use it to coordinate whether a request is idempotent, whether it's still valid or expired based on timestamps, etc. + +
+```mermaid +classDiagram + direction LR + class IdempotencyRecord { + idempotencyKey string + status Status + expiryTimestamp int + inProgressExpiryTimestamp int + responseData Json~str~ + payloadHash str + } + class Status { + <> + INPROGRESS + COMPLETE + EXPIRED internal_only + } + IdempotencyRecord -- Status +``` + +Idempotency record representation +
+ +## Key features + +* Prevent Lambda handler from executing more than once on the same event payload during a time window +* Ensure Lambda handler returns the same result when called with the same payload +* Select a subset of the event as the idempotency key using JMESPath expressions +* Set a time window in which records with the same payload should be considered duplicates +* Expires in-progress executions if the Lambda function times out halfway through + +## Getting started + +### IAM Permissions + +Your Lambda function IAM Role must have `dynamodb:GetItem`, `dynamodb:PutItem`, `dynamodb:UpdateItem` and `dynamodb:DeleteItem` IAM permissions before using this feature. + +???+ note + If you're using our example [AWS Serverless Application Model (SAM)](#required-resources), it already adds the required permissions. + +### Required resources + +Before getting started, you need to create a persistent storage layer where the idempotency utility can store its state - your lambda functions will need read and write access to it. + +As of now, Amazon DynamoDB is the only supported persistent storage layer, so you'll need to create a table first. + +**Default table configuration** + +If you're not [changing the default configuration for the DynamoDB persistence layer](#dynamodbpersistencelayer), this is the expected default configuration: + +| Configuration | Value | Notes | +| ------------------ | ------------ | ----------------------------------------------------------------------------------- | +| Partition key | `id` | +| TTL attribute name | `expiration` | This can only be configured after your table is created if you're using AWS Console | + + +???+ tip "Tip: You can share a single state table for all functions" + You can reuse the same DynamoDB table to store idempotency state. We add `module_name` and [qualified name for classes and functions](https://peps.python.org/pep-3155/) in addition to the idempotency key as a hash key. + +```yaml hl_lines="5-13 21-23" title="AWS Serverless Application Model (SAM) example" +Resources: + IdempotencyTable: + Type: AWS::DynamoDB::Table + Properties: + AttributeDefinitions: + - AttributeName: id + AttributeType: S + KeySchema: + - AttributeName: id + KeyType: HASH + TimeToLiveSpecification: + AttributeName: expiration + Enabled: true + BillingMode: PAY_PER_REQUEST + + HelloWorldFunction: + Type: AWS::Serverless::Function + Properties: + Runtime: nodejs18.x + Policies: + - DynamoDBCrudPolicy: + TableName: !Ref IdempotencyTable +``` + +??? warning "Warning: Large responses with DynamoDB persistence layer" + When using this utility with DynamoDB, your function's responses must be [smaller than 400KB](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Limits.html#limits-items). + + Larger items cannot be written to DynamoDB and will cause exceptions. + +???+ info "Info: DynamoDB" + Each function invocation will generally make 2 requests to DynamoDB. If the + result returned by your Lambda is less than 1kb, you can expect 2 WCUs per invocation. For retried invocations, you will see 1WCU and 1RCU. Review the [DynamoDB pricing documentation](https://aws.amazon.com/dynamodb/pricing/) to + estimate the cost. + +### IdempotentLambdaHandler decorator + +You can quickly start by initializing the `DynamoDBPersistenceLayer` class and using it with the `idempotent` decorator on your Lambda handler. + +???+ note + In this example, the entire Lambda handler is treated as a single idempotent operation. If your Lambda handler can cause multiple side effects, or you're only interested in making a specific logic idempotent, use the [`idempotentMethod` decorator](#idempotentMethod-decorator) or the [`makeFunctionIdempotent` high-order function](#makeFunctionIdempotent-high-order-function) instead. + +!!! tip "See [Choosing a payload subset for idempotency](#choosing-a-payload-subset-for-idempotency) for more elaborate use cases." + +=== "index.ts" + + ```typescript hl_lines="1-2 6-8 12-14 23" + --8<-- "docs/snippets/idempotency/idempotentLambdaHandler.ts" + ``` + + 1. Binding your handler method allows your handler to access `this` within the class methods. + +=== "Example event" + + ```json + { + "username": "xyz", + "productId": "123456789" + } + ``` + +After processing this request successfully, a second request containing the exact same payload above will now return the same response, ensuring our customer isn't charged twice. + +!!! question "New to idempotency concept? Please review our [Terminology](#terminology) section if you haven't yet." + +### IdempotentMethod decorator + +Similar to [idempotent decorator](#idempotent-decorator), you can use `idempotent_function` decorator for any synchronous Python function. + +When using `idempotent_function`, you must tell us which keyword parameter in your function signature has the data we should use via **`data_keyword_argument`**. + +!!! tip "We support JSON serializable data, [Python Dataclasses](https://docs.python.org/3.7/library/dataclasses.html){target="_blank"}, [Parser/Pydantic Models](parser.md){target="_blank"}, and our [Event Source Data Classes](./data_classes.md){target="_blank"}." + +???+ warning "Limitation" + Make sure to call your decorated function using keyword arguments. + +=== "dataclass_sample.py" + + ```python hl_lines="3-4 23 33" + from dataclasses import dataclass + + from aws_lambda_powertools.utilities.idempotency import ( + DynamoDBPersistenceLayer, IdempotencyConfig, idempotent_function) + + dynamodb = DynamoDBPersistenceLayer(table_name="idem") + config = IdempotencyConfig( + event_key_jmespath="order_id", # see Choosing a payload subset section + use_local_cache=True, + ) + + @dataclass + class OrderItem: + sku: str + description: str + + @dataclass + class Order: + item: OrderItem + order_id: int + + + @idempotent_function(data_keyword_argument="order", config=config, persistence_store=dynamodb) + def process_order(order: Order): + return f"processed order {order.order_id}" + + def lambda_handler(event, context): + config.register_lambda_context(context) # see Lambda timeouts section + order_item = OrderItem(sku="fake", description="sample") + order = Order(item=order_item, order_id="fake-id") + + # `order` parameter must be called as a keyword argument to work + process_order(order=order) + ``` + +=== "parser_pydantic_sample.py" + + ```python hl_lines="1-2 22 32" + from aws_lambda_powertools.utilities.idempotency import ( + DynamoDBPersistenceLayer, IdempotencyConfig, idempotent_function) + from aws_lambda_powertools.utilities.parser import BaseModel + + dynamodb = DynamoDBPersistenceLayer(table_name="idem") + config = IdempotencyConfig( + event_key_jmespath="order_id", # see Choosing a payload subset section + use_local_cache=True, + ) + + + class OrderItem(BaseModel): + sku: str + description: str + + + class Order(BaseModel): + item: OrderItem + order_id: int + + + @idempotent_function(data_keyword_argument="order", config=config, persistence_store=dynamodb) + def process_order(order: Order): + return f"processed order {order.order_id}" + + def lambda_handler(event, context): + config.register_lambda_context(context) # see Lambda timeouts section + order_item = OrderItem(sku="fake", description="sample") + order = Order(item=order_item, order_id="fake-id") + + # `order` parameter must be called as a keyword argument to work + process_order(order=order) + ``` + +### makeFunctionIdempotent higher-order function + +### Choosing a payload subset for idempotency + +???+ tip "Tip: Dealing with always changing payloads" + When dealing with a more elaborate payload, where parts of the payload always change, you should use the **`eventKeyJmesPath`** parameter. + +Use [`IdempotencyConfig`](#customizing-the-default-behavior) to instruct the idempotent decorator to only use a portion of your payload to verify whether a request is idempotent, and therefore it should not be retried. + +> **Payment scenario** + +In this example, we have a Lambda handler that creates a payment for a user subscribing to a product. We want to ensure that we don't accidentally charge our customer by subscribing them more than once. + +Imagine the function executes successfully, but the client never receives the response due to a connection issue. It is safe to retry in this instance, as the idempotent decorator will return a previously saved response. + +**What we want here** is to instruct Idempotency to use the `user` and `productId` fields from our incoming payload as our idempotency key. If we were to treat the entire request as our idempotency key, a simple HTTP header or timestamp change would cause our customer to be charged twice. + +???+ tip "Deserializing JSON strings in payloads for increased accuracy." + The payload extracted by the `eventKeyJmesPath` is treated as a string by default. This means there could be differences in whitespace even when the JSON payload itself is identical. + + To alter this behaviour, we can use the [JMESPath built-in function](jmespath_functions.md#powertools_json-function) `powertools_json()` to treat the payload as a JSON object (dict) rather than a string. + +=== "payment.py" + + ```python hl_lines="2-4 10 12 15 20" + import json + from aws_lambda_powertools.utilities.idempotency import ( + IdempotencyConfig, DynamoDBPersistenceLayer, idempotent + ) + + persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") + + # Deserialize JSON string under the "body" key + # then extract "user" and "product_id" data + config = IdempotencyConfig(event_key_jmespath="powertools_json(body).[user, product_id]") + + @idempotent(config=config, persistence_store=persistence_layer) + def handler(event, context): + body = json.loads(event['body']) + payment = create_subscription_payment( + user=body['user'], + product=body['product_id'] + ) + ... + return { + "payment_id": payment.id, + "message": "success", + "statusCode": 200 + } + ``` + +=== "Example event" + + ```json hl_lines="28" + { + "version":"2.0", + "routeKey":"ANY /createpayment", + "rawPath":"/createpayment", + "rawQueryString":"", + "headers": { + "Header1": "value1", + "Header2": "value2" + }, + "requestContext":{ + "accountId":"123456789012", + "apiId":"api-id", + "domainName":"id.execute-api.us-east-1.amazonaws.com", + "domainPrefix":"id", + "http":{ + "method":"POST", + "path":"/createpayment", + "protocol":"HTTP/1.1", + "sourceIp":"ip", + "userAgent":"agent" + }, + "requestId":"id", + "routeKey":"ANY /createpayment", + "stage":"$default", + "time":"10/Feb/2021:13:40:43 +0000", + "timeEpoch":1612964443723 + }, + "body":"{\"user\":\"xyz\",\"product_id\":\"123456789\"}", + "isBase64Encoded":false + } + ``` + +### Lambda timeouts + +???+ note + This is automatically done when you decorate your Lambda handler with [@idempotent decorator](#idempotent-decorator). + +To prevent against extended failed retries when a [Lambda function times out](https://aws.amazon.com/premiumsupport/knowledge-center/lambda-verify-invocation-timeouts/), Powertools for AWS Lambda (Python) calculates and includes the remaining invocation available time as part of the idempotency record. + +???+ example + If a second invocation happens **after** this timestamp, and the record is marked as `INPROGRESS`, we will execute the invocation again as if it was in the `EXPIRED` state (e.g, `expire_seconds` field elapsed). + + This means that if an invocation expired during execution, it will be quickly executed again on the next retry. + +???+ important + If you are only using the [@idempotent_function decorator](#idempotent_function-decorator) to guard isolated parts of your code, you must use `register_lambda_context` available in the [idempotency config object](#customizing-the-default-behavior) to benefit from this protection. + +Here is an example on how you register the Lambda context in your handler: + +```python hl_lines="8 16" title="Registering the Lambda context" +from aws_lambda_powertools.utilities.data_classes.sqs_event import SQSRecord +from aws_lambda_powertools.utilities.idempotency import ( + IdempotencyConfig, idempotent_function +) + +persistence_layer = DynamoDBPersistenceLayer(table_name="...") + +config = IdempotencyConfig() + +@idempotent_function(data_keyword_argument="record", persistence_store=persistence_layer, config=config) +def record_handler(record: SQSRecord): + return {"message": record["body"]} + + +def lambda_handler(event, context): + config.register_lambda_context(context) + + return record_handler(event) +``` + +### Handling exceptions + +If you are using the `idempotent` decorator on your Lambda handler, any unhandled exceptions that are raised during the code execution will cause **the record in the persistence layer to be deleted**. +This means that new invocations will execute your code again despite having the same payload. If you don't want the record to be deleted, you need to catch exceptions within the idempotent function and return a successful response. + +
+```mermaid +sequenceDiagram + participant Client + participant Lambda + participant Persistence Layer + Client->>Lambda: Invoke (event) + Lambda->>Persistence Layer: Get or set (id=event.search(payload)) + activate Persistence Layer + Note right of Persistence Layer: Locked during this time. Prevents multiple
Lambda invocations with the same
payload running concurrently. + Lambda--xLambda: Call handler (event).
Raises exception + Lambda->>Persistence Layer: Delete record (id=event.search(payload)) + deactivate Persistence Layer + Lambda-->>Client: Return error response +``` +Idempotent sequence exception +
+ +If you are using `idempotent_function`, any unhandled exceptions that are raised _inside_ the decorated function will cause the record in the persistence layer to be deleted, and allow the function to be executed again if retried. + +If an Exception is raised _outside_ the scope of the decorated function and after your function has been called, the persistent record will not be affected. In this case, idempotency will be maintained for your decorated function. Example: + +```python hl_lines="2-4 8-10" title="Exception not affecting idempotency record sample" +def lambda_handler(event, context): + # If an exception is raised here, no idempotent record will ever get created as the + # idempotent function does not get called + do_some_stuff() + + result = call_external_service(data={"user": "user1", "id": 5}) + + # This exception will not cause the idempotent record to be deleted, since it + # happens after the decorated function has been successfully called + raise Exception + + +@idempotent_function(data_keyword_argument="data", config=config, persistence_store=dynamodb) +def call_external_service(data: dict, **kwargs): + result = requests.post('http://example.com', json={"user": data['user'], "transaction_id": data['id']} + return result.json() +``` + +???+ warning + **We will raise `IdempotencyPersistenceLayerError`** if any of the calls to the persistence layer fail unexpectedly. + + As this happens outside the scope of your decorated function, you are not able to catch it if you're using the `idempotent` decorator on your Lambda handler. + +### Idempotency request flow + +The following sequence diagrams explain how the Idempotency feature behaves under different scenarios. + +#### Successful request + +
+```mermaid +sequenceDiagram + participant Client + participant Lambda + participant Persistence Layer + alt initial request + Client->>Lambda: Invoke (event) + Lambda->>Persistence Layer: Get or set idempotency_key=hash(payload) + activate Persistence Layer + Note over Lambda,Persistence Layer: Set record status to INPROGRESS.
Prevents concurrent invocations
with the same payload + Lambda-->>Lambda: Call your function + Lambda->>Persistence Layer: Update record with result + deactivate Persistence Layer + Persistence Layer-->>Persistence Layer: Update record + Note over Lambda,Persistence Layer: Set record status to COMPLETE.
New invocations with the same payload
now return the same result + Lambda-->>Client: Response sent to client + else retried request + Client->>Lambda: Invoke (event) + Lambda->>Persistence Layer: Get or set idempotency_key=hash(payload) + activate Persistence Layer + Persistence Layer-->>Lambda: Already exists in persistence layer. + deactivate Persistence Layer + Note over Lambda,Persistence Layer: Record status is COMPLETE and not expired + Lambda-->>Client: Same response sent to client + end +``` +Idempotent successful request +
+ +#### Successful request with cache enabled + +!!! note "[In-memory cache is disabled by default](#using-in-memory-cache)." + +
+```mermaid +sequenceDiagram + participant Client + participant Lambda + participant Persistence Layer + alt initial request + Client->>Lambda: Invoke (event) + Lambda->>Persistence Layer: Get or set idempotency_key=hash(payload) + activate Persistence Layer + Note over Lambda,Persistence Layer: Set record status to INPROGRESS.
Prevents concurrent invocations
with the same payload + Lambda-->>Lambda: Call your function + Lambda->>Persistence Layer: Update record with result + deactivate Persistence Layer + Persistence Layer-->>Persistence Layer: Update record + Note over Lambda,Persistence Layer: Set record status to COMPLETE.
New invocations with the same payload
now return the same result + Lambda-->>Lambda: Save record and result in memory + Lambda-->>Client: Response sent to client + else retried request + Client->>Lambda: Invoke (event) + Lambda-->>Lambda: Get idempotency_key=hash(payload) + Note over Lambda,Persistence Layer: Record status is COMPLETE and not expired + Lambda-->>Client: Same response sent to client + end +``` +Idempotent successful request cached +
+ +#### Expired idempotency records + +
+```mermaid +sequenceDiagram + participant Client + participant Lambda + participant Persistence Layer + alt initial request + Client->>Lambda: Invoke (event) + Lambda->>Persistence Layer: Get or set idempotency_key=hash(payload) + activate Persistence Layer + Note over Lambda,Persistence Layer: Set record status to INPROGRESS.
Prevents concurrent invocations
with the same payload + Lambda-->>Lambda: Call your function + Lambda->>Persistence Layer: Update record with result + deactivate Persistence Layer + Persistence Layer-->>Persistence Layer: Update record + Note over Lambda,Persistence Layer: Set record status to COMPLETE.
New invocations with the same payload
now return the same result + Lambda-->>Client: Response sent to client + else retried request + Client->>Lambda: Invoke (event) + Lambda->>Persistence Layer: Get or set idempotency_key=hash(payload) + activate Persistence Layer + Persistence Layer-->>Lambda: Already exists in persistence layer. + deactivate Persistence Layer + Note over Lambda,Persistence Layer: Record status is COMPLETE but expired hours ago + loop Repeat initial request process + Note over Lambda,Persistence Layer: 1. Set record to INPROGRESS,
2. Call your function,
3. Set record to COMPLETE + end + Lambda-->>Client: Same response sent to client + end +``` +Previous Idempotent request expired +
+ +#### Concurrent identical in-flight requests + +
+```mermaid +sequenceDiagram + participant Client + participant Lambda + participant Persistence Layer + Client->>Lambda: Invoke (event) + Lambda->>Persistence Layer: Get or set idempotency_key=hash(payload) + activate Persistence Layer + Note over Lambda,Persistence Layer: Set record status to INPROGRESS.
Prevents concurrent invocations
with the same payload + par Second request + Client->>Lambda: Invoke (event) + Lambda->>Persistence Layer: Get or set idempotency_key=hash(payload) + Lambda--xLambda: IdempotencyAlreadyInProgressError + Lambda->>Client: Error sent to client if unhandled + end + Lambda-->>Lambda: Call your function + Lambda->>Persistence Layer: Update record with result + deactivate Persistence Layer + Persistence Layer-->>Persistence Layer: Update record + Note over Lambda,Persistence Layer: Set record status to COMPLETE.
New invocations with the same payload
now return the same result + Lambda-->>Client: Response sent to client +``` +Concurrent identical in-flight requests +
+ +#### Lambda request timeout + +
+```mermaid +sequenceDiagram + participant Client + participant Lambda + participant Persistence Layer + alt initial request + Client->>Lambda: Invoke (event) + Lambda->>Persistence Layer: Get or set idempotency_key=hash(payload) + activate Persistence Layer + Note over Lambda,Persistence Layer: Set record status to INPROGRESS.
Prevents concurrent invocations
with the same payload + Lambda-->>Lambda: Call your function + Note right of Lambda: Time out + Lambda--xLambda: Time out error + Lambda-->>Client: Return error response + deactivate Persistence Layer + else retry after Lambda timeout elapses + Client->>Lambda: Invoke (event) + Lambda->>Persistence Layer: Get or set idempotency_key=hash(payload) + activate Persistence Layer + Note over Lambda,Persistence Layer: Set record status to INPROGRESS.
Reset in_progress_expiry attribute + Lambda-->>Lambda: Call your function + Lambda->>Persistence Layer: Update record with result + deactivate Persistence Layer + Persistence Layer-->>Persistence Layer: Update record + Lambda-->>Client: Response sent to client + end +``` +Idempotent request during and after Lambda timeouts +
+ +## Advanced + +### Persistence layers + +#### DynamoDBPersistenceLayer + +This persistence layer is built-in, and you can either use an existing DynamoDB table or create a new one dedicated for idempotency state (recommended). + +```python hl_lines="5-10" title="Customizing DynamoDBPersistenceLayer to suit your table structure" +from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer + +persistence_layer = DynamoDBPersistenceLayer( + table_name="IdempotencyTable", + key_attr="idempotency_key", + expiry_attr="expires_at", + in_progress_expiry_attr="in_progress_expires_at", + status_attr="current_status", + data_attr="result_data", + validation_key_attr="validation_key", +) +``` + +When using DynamoDB as a persistence layer, you can alter the attribute names by passing these parameters when initializing the persistence layer: + +| Parameter | Required | Default | Description | +| --------------------------- | ------------------ | ------------------------------------ | -------------------------------------------------------------------------------------------------------- | +| **table_name** | :heavy_check_mark: | | Table name to store state | +| **key_attr** | | `id` | Partition key of the table. Hashed representation of the payload (unless **sort_key_attr** is specified) | +| **expiry_attr** | | `expiration` | Unix timestamp of when record expires | +| **in_progress_expiry_attr** | | `in_progress_expiration` | Unix timestamp of when record expires while in progress (in case of the invocation times out) | +| **status_attr** | | `status` | Stores status of the lambda execution during and after invocation | +| **data_attr** | | `data` | Stores results of successfully executed Lambda handlers | +| **validation_key_attr** | | `validation` | Hashed representation of the parts of the event used for validation | +| **sort_key_attr** | | | Sort key of the table (if table is configured with a sort key). | +| **static_pk_value** | | `idempotency#{LAMBDA_FUNCTION_NAME}` | Static value to use as the partition key. Only used when **sort_key_attr** is set. | + +### Customizing the default behavior + +Idempotent decorator can be further configured with **`IdempotencyConfig`** as seen in the previous example. These are the available options for further configuration + +| Parameter | Default | Description | +| ------------------------------- | ------- | ----------------------------------------------------------------------------------------------------------------------------------------- | +| **event_key_jmespath** | `""` | JMESPath expression to extract the idempotency key from the event record using [built-in functions](/utilities/jmespath_functions) | +| **payload_validation_jmespath** | `""` | JMESPath expression to validate whether certain parameters have changed in the event while the event payload | +| **raise_on_no_idempotency_key** | `False` | Raise exception if no idempotency key was found in the request | +| **expires_after_seconds** | 3600 | The number of seconds to wait before a record is expired | +| **use_local_cache** | `False` | Whether to locally cache idempotency results | +| **local_cache_max_items** | 256 | Max number of items to store in local cache | +| **hash_function** | `md5` | Function to use for calculating hashes, as provided by [hashlib](https://docs.python.org/3/library/hashlib.html) in the standard library. | + +### Handling concurrent executions with the same payload + +This utility will raise an **`IdempotencyAlreadyInProgressError`** exception if you receive **multiple invocations with the same payload while the first invocation hasn't completed yet**. + +???+ info + If you receive `IdempotencyAlreadyInProgressError`, you can safely retry the operation. + +This is a locking mechanism for correctness. Since we don't know the result from the first invocation yet, we can't safely allow another concurrent execution. + +### Using in-memory cache + +**By default, in-memory local caching is disabled**, since we don't know how much memory you consume per invocation compared to the maximum configured in your Lambda function. + +???+ note "Note: This in-memory cache is local to each Lambda execution environment" + This means it will be effective in cases where your function's concurrency is low in comparison to the number of "retry" invocations with the same payload, because cache might be empty. + +You can enable in-memory caching with the **`use_local_cache`** parameter: + +```python hl_lines="8 11" title="Caching idempotent transactions in-memory to prevent multiple calls to storage" +from aws_lambda_powertools.utilities.idempotency import ( + IdempotencyConfig, DynamoDBPersistenceLayer, idempotent +) + +persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") +config = IdempotencyConfig( + event_key_jmespath="body", + use_local_cache=True, +) + +@idempotent(config=config, persistence_store=persistence_layer) +def handler(event, context): + ... +``` + +When enabled, the default is to cache a maximum of 256 records in each Lambda execution environment - You can change it with the **`local_cache_max_items`** parameter. + +### Expiring idempotency records + +!!! note "By default, we expire idempotency records after **an hour** (3600 seconds)." + +In most cases, it is not desirable to store the idempotency records forever. Rather, you want to guarantee that the same payload won't be executed within a period of time. + +You can change this window with the **`expires_after_seconds`** parameter: + +```python hl_lines="8 11" title="Adjusting idempotency record expiration" +from aws_lambda_powertools.utilities.idempotency import ( + IdempotencyConfig, DynamoDBPersistenceLayer, idempotent +) + +persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") +config = IdempotencyConfig( + event_key_jmespath="body", + expires_after_seconds=5*60, # 5 minutes +) + +@idempotent(config=config, persistence_store=persistence_layer) +def handler(event, context): + ... +``` + +This will mark any records older than 5 minutes as expired, and [your function will be executed as normal if it is invoked with a matching payload](#expired-idempotency-records). + +???+ important "Idempotency record expiration vs DynamoDB time-to-live (TTL)" + [DynamoDB TTL is a feature](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/howitworks-ttl.html){target="_blank"} to remove items after a certain period of time, it may occur within 48 hours of expiration. + + We don't rely on DynamoDB or any persistence storage layer to determine whether a record is expired to avoid eventual inconsistency states. + + Instead, Idempotency records saved in the storage layer contain timestamps that can be verified upon retrieval and double checked within Idempotency feature. + + **Why?** + + A record might still be valid (`COMPLETE`) when we retrieved, but in some rare cases it might expire a second later. A record could also be [cached in memory](#using-in-memory-cache). You might also want to have idempotent transactions that should expire in seconds. + +### Payload validation + +???+ question "Question: What if your function is invoked with the same payload except some outer parameters have changed?" + Example: A payment transaction for a given productID was requested twice for the same customer, **however the amount to be paid has changed in the second transaction**. + +By default, we will return the same result as it returned before, however in this instance it may be misleading; we provide a fail fast payload validation to address this edge case. + +With **`payload_validation_jmespath`**, you can provide an additional JMESPath expression to specify which part of the event body should be validated against previous idempotent invocations + +=== "app.py" + + ```python hl_lines="7 11 18 25" + from aws_lambda_powertools.utilities.idempotency import ( + IdempotencyConfig, DynamoDBPersistenceLayer, idempotent + ) + + config = IdempotencyConfig( + event_key_jmespath="[userDetail, productId]", + payload_validation_jmespath="amount" + ) + persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") + + @idempotent(config=config, persistence_store=persistence_layer) + def handler(event, context): + # Creating a subscription payment is a side + # effect of calling this function! + payment = create_subscription_payment( + user=event['userDetail']['username'], + product=event['product_id'], + amount=event['amount'] + ) + ... + return { + "message": "success", + "statusCode": 200, + "payment_id": payment.id, + "amount": payment.amount + } + ``` + +=== "Example Event 1" + + ```json hl_lines="8" + { + "userDetail": { + "username": "User1", + "user_email": "user@example.com" + }, + "productId": 1500, + "charge_type": "subscription", + "amount": 500 + } + ``` + +=== "Example Event 2" + + ```json hl_lines="8" + { + "userDetail": { + "username": "User1", + "user_email": "user@example.com" + }, + "productId": 1500, + "charge_type": "subscription", + "amount": 1 + } + ``` + +In this example, the **`userDetail`** and **`productId`** keys are used as the payload to generate the idempotency key, as per **`event_key_jmespath`** parameter. + +???+ note + If we try to send the same request but with a different amount, we will raise **`IdempotencyValidationError`**. + +Without payload validation, we would have returned the same result as we did for the initial request. Since we're also returning an amount in the response, this could be quite confusing for the client. + +By using **`payload_validation_jmespath="amount"`**, we prevent this potentially confusing behavior and instead raise an Exception. + +### Making idempotency key required +{ +const If you new want to enforce that an idempotency key is required, you can set **`throwOnNoIdempotencyKey: t* to `True`.} +; +This means that we will raise **`IdempotencyKeyError`** if the evaluation of **`event_key_jmespath`** is `None`. + +=== "app.py" + + ```typescript hl_lines="9-10 13" + + ``` + +=== "Success Event" + + ```json hl_lines="3 6" + { + "user": { + "uid": "BB0D045C-8878-40C8-889E-38B3CB0A61B1", + "name": "foo bar" + }, + "orderId": 10000 + } + ``` + +=== "Failure Event" + + Notice that `orderId` is now accidentally within the `user` key + + ```json hl_lines="3 5" + { + "user": { + "uid": "BB0D045C-8878-40C8-889E-38B3CB0A61B1", + "name": "foo bar", + "orderId": 10000 + }, + } + ``` + +### Customizing boto configuration + +The **`boto_config`** and **`boto3_session`** parameters enable you to pass in a custom [botocore config object](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html) or a custom [boto3 session](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html) when constructing the persistence store. + +=== "Custom session" + + ```python hl_lines="1 6 9 14" + import boto3 + from aws_lambda_powertools.utilities.idempotency import ( + IdempotencyConfig, DynamoDBPersistenceLayer, idempotent + ) + + boto3_session = boto3.session.Session() + persistence_layer = DynamoDBPersistenceLayer( + table_name="IdempotencyTable", + boto3_session=boto3_session + ) + + config = IdempotencyConfig(event_key_jmespath="body") + + @idempotent(config=config, persistence_store=persistence_layer) + def handler(event, context): + ... + ``` +=== "Custom config" + + ```python hl_lines="1 7 10" + from botocore.config import Config + from aws_lambda_powertools.utilities.idempotency import ( + IdempotencyConfig, DynamoDBPersistenceLayer, idempotent + ) + + config = IdempotencyConfig(event_key_jmespath="body") + boto_config = Config() + persistence_layer = DynamoDBPersistenceLayer( + table_name="IdempotencyTable", + boto_config=boto_config + ) + + @idempotent(config=config, persistence_store=persistence_layer) + def handler(event, context): + ... + ``` + +### Using a DynamoDB table with a composite primary key + +When using a composite primary key table (hash+range key), use the `sortKeyAttr` parameter when initializing your persistence layer. + +With this setting, we will save the idempotency key in the sort key instead of the primary key. By default, the primary key will now be set to `idempotency#{LAMBDA_FUNCTION_NAME}`. + +You can optionally set a static value for the partition key using the `staticPkValue` parameter. + +```python hl_lines="5" title="Reusing a DynamoDB table that uses a composite primary key" +--8<-- "docs/snippets/idempotency/dynamoDBCompositePK.ts" +``` + + + +The example function above would cause data to be stored in DynamoDB like this: + +| id | sort_key | expiration | status | data | +| ---------------------------- | -------------------------------- | ---------- | ----------- | ------------------------------------ | +| idempotency#MyLambdaFunction | 1e956ef7da78d0cb890be999aecc0c9e | 1636549553 | COMPLETED | {"id": 12391, "message": "success"} | +| idempotency#MyLambdaFunction | 2b2cdb5f86361e97b4383087c1ffdf27 | 1636549571 | COMPLETED | {"id": 527212, "message": "success"} | +| idempotency#MyLambdaFunction | f091d2527ad1c78f05d54cc3f363be80 | 1636549585 | IN_PROGRESS | | + + + +## Testing your code + +The idempotency utility provides several routes to test your code. + +### Disabling the idempotency utility + +When testing your code, you may wish to disable the idempotency logic altogether and focus on testing your business logic. To do this, you can set the environment variable `POWERTOOLS_IDEMPOTENCY_DISABLED` with a truthy value. + +=== "index.test.ts" + + ```python hl_lines="16" + --8<-- "docs/snippets/idempotency/disablingUtility.ts" + ``` + +=== "index.ts" + + ```typescript + --8<-- "docs/snippets/idempotency/testingYourCodeFunction.ts" + ``` + +### Testing with DynamoDB Local + +To test with [DynamoDB Local](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocal.DownloadingAndRunning.html), you can replace the `DynamoDBClient` used by the persistence layer with one you create inside your tests. This allows you to set a local endpoint. + +=== "index.test.ts" + + ```python + --8<-- "docs/snippets/idempotency/localDynamoDB.ts" + ``` + +=== "index.ts" + + ```typescript + --8<-- "docs/snippets/idempotency/testingYourCodeFunction.ts" + ``` + +### How do I mock all DynamoDB I/O operations + +The idempotency utility lazily creates an AWS SDK for JavaScript v3 [`DynamoDBClient`](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-dynamodb/index.html) which it uses to access DynamoDB. This means it is possible to mock or spy on the API calls. + +=== "index.test.ts" + + ```python + --8<-- "docs/snippets/idempotency/jestMockDynamoDB.ts" + ``` + +=== "index.ts" + + ```typescript + --8<-- "docs/snippets/idempotency/testingYourCodeFunction.ts" + ``` + +## Extra resources + +If you're interested in a deep dive on how Amazon uses idempotency when building our APIs, check out +[this article](https://aws.amazon.com/builders-library/making-retries-safe-with-idempotent-APIs/). \ No newline at end of file From e626f7c8fbf5828f2a55473af082e3e280fb5e82 Mon Sep 17 00:00:00 2001 From: Andrea Amorosi Date: Thu, 29 Jun 2023 14:13:54 +0200 Subject: [PATCH 2/7] wip --- .../idempotency/idempotentLambdaHandler.ts | 44 ----- docs/snippets/idempotency/idempotentMethod.ts | 4 - .../idempotency/makeFunctionIdempotentBase.ts | 16 ++ docs/utilities/idempotency.md | 160 ++++++++++++++---- packages/idempotency/README.md | 2 +- 5 files changed, 146 insertions(+), 80 deletions(-) delete mode 100644 docs/snippets/idempotency/idempotentLambdaHandler.ts delete mode 100644 docs/snippets/idempotency/idempotentMethod.ts create mode 100644 docs/snippets/idempotency/makeFunctionIdempotentBase.ts diff --git a/docs/snippets/idempotency/idempotentLambdaHandler.ts b/docs/snippets/idempotency/idempotentLambdaHandler.ts deleted file mode 100644 index ba20d4839c..0000000000 --- a/docs/snippets/idempotency/idempotentLambdaHandler.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { idempotentLambdaHandler } from '@aws-lambda-powertools/idempotency'; -import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; -import { LambdaInterface } from '@aws-lambda-powertools/commons'; -import type { Request, Response, SubscriptionResult } from './types'; - -const persistenceStore = new DynamoDBPersistenceLayer({ - tableName: 'IdempotencyTable', -}); - -class Lambda implements LambdaInterface { - // Decorate your handler class method - @idempotentLambdaHandler({ - persistenceStore, - }) - public async handler(event: Request, _context: unknown): Promise { - const payment = await this.#createSubscriptionPayment( - event.user, - event.productId - ); - - await this.#sendNotification(event.email); - - return { - paymentId: payment.id, - statusCode: 200, - }; - } - - async #createSubscriptionPayment( - _user: string, - _product: string - ): Promise { - return { - id: 'foo', - }; - } - - async #sendNotification(_email: string): Promise { - // ... - } -} - -const myFunction = new Lambda(); -export const handler = myFunction.handler.bind(myFunction); // (1) diff --git a/docs/snippets/idempotency/idempotentMethod.ts b/docs/snippets/idempotency/idempotentMethod.ts deleted file mode 100644 index 7699ba4048..0000000000 --- a/docs/snippets/idempotency/idempotentMethod.ts +++ /dev/null @@ -1,4 +0,0 @@ -import { idempotentLambdaHandler } from '@aws-lambda-powertools/idempotency'; -import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; -import { LambdaInterface } from '@aws-lambda-powertools/commons'; -import type { Request, Response, SubscriptionResult } from './types'; \ No newline at end of file diff --git a/docs/snippets/idempotency/makeFunctionIdempotentBase.ts b/docs/snippets/idempotency/makeFunctionIdempotentBase.ts new file mode 100644 index 0000000000..83d50628c1 --- /dev/null +++ b/docs/snippets/idempotency/makeFunctionIdempotentBase.ts @@ -0,0 +1,16 @@ +import { makeFunctionIdempotent } from '@aws-lambda-powertools/idempotency'; +import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; +import type { Context } from 'aws-lambda'; + +const persistenceStore = new DynamoDBPersistenceLayer({ + tableName: 'idempotencyTableName', +}); + +export const handler = makeFunctionIdempotent( + async (_event: unknown, _context: unknown): Promise => { + // your code goes here here + }, + { + persistenceStore, + } +); diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index b2785de601..cb5d36cd3c 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -57,7 +57,7 @@ classDiagram Your Lambda function IAM Role must have `dynamodb:GetItem`, `dynamodb:PutItem`, `dynamodb:UpdateItem` and `dynamodb:DeleteItem` IAM permissions before using this feature. ???+ note - If you're using our example [AWS Serverless Application Model (SAM)](#required-resources), it already adds the required permissions. + If you're using one of our [examples below](#required-resources) the required permissions are already included. ### Required resources @@ -78,30 +78,130 @@ If you're not [changing the default configuration for the DynamoDB persistence l ???+ tip "Tip: You can share a single state table for all functions" You can reuse the same DynamoDB table to store idempotency state. We add `module_name` and [qualified name for classes and functions](https://peps.python.org/pep-3155/) in addition to the idempotency key as a hash key. -```yaml hl_lines="5-13 21-23" title="AWS Serverless Application Model (SAM) example" -Resources: - IdempotencyTable: - Type: AWS::DynamoDB::Table - Properties: - AttributeDefinitions: - - AttributeName: id - AttributeType: S - KeySchema: - - AttributeName: id - KeyType: HASH - TimeToLiveSpecification: - AttributeName: expiration - Enabled: true - BillingMode: PAY_PER_REQUEST - - HelloWorldFunction: - Type: AWS::Serverless::Function - Properties: - Runtime: nodejs18.x - Policies: - - DynamoDBCrudPolicy: - TableName: !Ref IdempotencyTable -``` + +=== "AWS Serverless Application Model (SAM) example" + + ```yaml hl_lines="6-14 24-31" + Transform: AWS::Serverless-2016-10-31 + Resources: + IdempotencyTable: + Type: AWS::DynamoDB::Table + Properties: + AttributeDefinitions: + - AttributeName: id + AttributeType: S + KeySchema: + - AttributeName: id + KeyType: HASH + TimeToLiveSpecification: + AttributeName: expiration + Enabled: true + BillingMode: PAY_PER_REQUEST + + HelloWorldFunction: + Type: AWS::Serverless::Function + Properties: + Runtime: nodejs18.x + Handler: index.handler + Policies: + - Statement: + - Sid: AllowDynamodbReadWrite + Effect: Allow + Action: + - dynamodb:PutItem + - dynamodb:GetItem + - dynamodb:UpdateItem + - dynamodb:DeleteItem + Resource: !GetAtt IdempotencyTable.Arn + ``` +=== "AWS Cloud Development Kit (CDK)" + + ```typescript hl_lines="10 13 16 19-21" + import { Table, Attribute } from 'aws-cdk-lib/aws-dynamodb'; + ``` + +=== "Terraform" + + ```terraform hl_lines="14-26 64-70" + terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.0" + } + } + } + + provider "aws" { + region = "us-east-1" # Replace with your desired AWS region + } + + resource "aws_dynamodb_table" "IdempotencyTable" { + name = "IdempotencyTable" + billing_mode = "PAY_PER_REQUEST" + hash_key = "id" + attribute { + name = "id" + type = "S" + } + ttl { + attribute_name = "expiration" + enabled = true + } + } + + resource "aws_lambda_function" "IdempotencyFunction" { + function_name = "IdempotencyFunction" + role = aws_iam_role.IdempotencyFunctionRole.arn + runtime = "python3.10" + handler = "app.lambda_handler" + filename = "lambda.zip" + } + + resource "aws_iam_role" "IdempotencyFunctionRole" { + name = "IdempotencyFunctionRole" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "" + Effect = "Allow" + Principal = { + Service = "lambda.amazonaws.com" + } + Action = "sts:AssumeRole" + }, + ] + }) + } + + resource "aws_iam_policy" "LambdaDynamoDBPolicy" { + name = "LambdaDynamoDBPolicy" + description = "IAM policy for Lambda function to access DynamoDB" + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "AllowDynamodbReadWrite" + Effect = "Allow" + Action = [ + "dynamodb:PutItem", + "dynamodb:GetItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + ] + Resource = aws_dynamodb_table.IdempotencyTable.arn + }, + ] + }) + } + + resource "aws_iam_role_policy_attachment" "IdempotencyFunctionRoleAttachment" { + role = aws_iam_role.IdempotencyFunctionRole.name + policy_arn = aws_iam_policy.LambdaDynamoDBPolicy.arn + } + ``` ??? warning "Warning: Large responses with DynamoDB persistence layer" When using this utility with DynamoDB, your function's responses must be [smaller than 400KB](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Limits.html#limits-items). @@ -113,23 +213,21 @@ Resources: result returned by your Lambda is less than 1kb, you can expect 2 WCUs per invocation. For retried invocations, you will see 1WCU and 1RCU. Review the [DynamoDB pricing documentation](https://aws.amazon.com/dynamodb/pricing/) to estimate the cost. -### IdempotentLambdaHandler decorator +### MakeFunctionIdempotent function wrapper -You can quickly start by initializing the `DynamoDBPersistenceLayer` class and using it with the `idempotent` decorator on your Lambda handler. +You can quickly start by initializing the `DynamoDBPersistenceLayer` class and using it with the `makeFunctionIdempotent` function wrapper on your Lambda handler. ???+ note - In this example, the entire Lambda handler is treated as a single idempotent operation. If your Lambda handler can cause multiple side effects, or you're only interested in making a specific logic idempotent, use the [`idempotentMethod` decorator](#idempotentMethod-decorator) or the [`makeFunctionIdempotent` high-order function](#makeFunctionIdempotent-high-order-function) instead. + In this example, the entire Lambda handler is treated as a single idempotent operation. If your Lambda handler can cause multiple side effects, or you're only interested in making a specific logic idempotent, use the `makeFunctionIdempotent` high-order function only on the specific logic you want to make idempotent instead. !!! tip "See [Choosing a payload subset for idempotency](#choosing-a-payload-subset-for-idempotency) for more elaborate use cases." === "index.ts" ```typescript hl_lines="1-2 6-8 12-14 23" - --8<-- "docs/snippets/idempotency/idempotentLambdaHandler.ts" + --8<-- "docs/snippets/idempotency/makeFunctionIdempotentBase.ts" ``` - 1. Binding your handler method allows your handler to access `this` within the class methods. - === "Example event" ```json diff --git a/packages/idempotency/README.md b/packages/idempotency/README.md index 5fb4acd204..21927dddc5 100644 --- a/packages/idempotency/README.md +++ b/packages/idempotency/README.md @@ -112,7 +112,7 @@ const config = new IdempotencyConfig({ }); export const handler = middy( - async (event: APIGatewayProxyEvent, _context: Context): Promise => { + async (_event: APIGatewayProxyEvent, _context: Context): Promise => { // your code goes here here } ).use( From aaa53900d8dc59d3d75ae59a6fc83a7f4ed52806 Mon Sep 17 00:00:00 2001 From: Andrea Amorosi Date: Sat, 8 Jul 2023 20:04:34 +0200 Subject: [PATCH 3/7] chore: added paths to snippets tsconfig --- docs/snippets/tsconfig.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/snippets/tsconfig.json b/docs/snippets/tsconfig.json index 6a2d536968..59dfbd1435 100644 --- a/docs/snippets/tsconfig.json +++ b/docs/snippets/tsconfig.json @@ -34,6 +34,12 @@ ], "@aws-lambda-powertools/idempotency/persistence": [ "../../packages/idempotency/lib/persistence" + ], + "@aws-lambda-powertools/idempotency": [ + "../../packages/idempotency/lib" + ], + "@aws-lambda-powertools/idempotency/middleware": [ + "../../packages/idempotency/lib/middleware" ] }, }, From c660dd0f6c58a378834e6d0d17571c36b88aeec1 Mon Sep 17 00:00:00 2001 From: Andrea Amorosi Date: Sat, 8 Jul 2023 20:06:50 +0200 Subject: [PATCH 4/7] chore: added page to docs menu --- mkdocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/mkdocs.yml b/mkdocs.yml index a0d03992e6..f5b7408590 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -16,6 +16,7 @@ nav: - core/metrics.md - Utilities: - utilities/parameters.md + - utilities/idempotency.md theme: name: material From 5a0952e4bca6d6eda4100373d07274bc8b479adc Mon Sep 17 00:00:00 2001 From: Andrea Amorosi Date: Mon, 10 Jul 2023 13:34:52 +0200 Subject: [PATCH 5/7] docs(idempotency): utility docs --- .../idempotency/customizePersistenceLayer.ts | 35 + docs/snippets/idempotency/disablingUtility.ts | 28 - docs/snippets/idempotency/jestMockDynamoDB.ts | 40 -- docs/snippets/idempotency/localDynamoDB.ts | 23 - .../idempotency/makeFunctionIdempotentBase.ts | 16 - .../idempotency/makeHandlerIdempotent.ts | 40 ++ .../idempotency/makeIdempotentAnyFunction.ts | 59 ++ .../idempotency/makeIdempotentBase.ts | 38 ++ .../idempotency/makeIdempotentJmes.ts | 51 ++ .../makeIdempotentLambdaContext.ts | 51 ++ .../idempotency/requiredIdempotencyKey.ts | 4 +- .../idempotency/testingYourCodeFunction.ts | 23 - docs/snippets/idempotency/types.ts | 2 +- ...positePK.ts => workingWithCompositeKey.ts} | 20 +- .../idempotency/workingWithCustomClient.ts | 32 + .../idempotency/workingWithCustomConfig.ts | 30 + .../idempotency/workingWithExceptions.ts | 59 ++ .../workingWithIdempotencyRequiredKey.ts | 36 + .../idempotency/workingWithLocalCache.ts | 35 + .../workingWithPayloadValidation.ts | 58 ++ .../workingWithRecordExpiration.ts | 35 + docs/utilities/idempotency.md | 615 ++++++------------ 22 files changed, 768 insertions(+), 562 deletions(-) create mode 100644 docs/snippets/idempotency/customizePersistenceLayer.ts delete mode 100644 docs/snippets/idempotency/disablingUtility.ts delete mode 100644 docs/snippets/idempotency/jestMockDynamoDB.ts delete mode 100644 docs/snippets/idempotency/localDynamoDB.ts delete mode 100644 docs/snippets/idempotency/makeFunctionIdempotentBase.ts create mode 100644 docs/snippets/idempotency/makeHandlerIdempotent.ts create mode 100644 docs/snippets/idempotency/makeIdempotentAnyFunction.ts create mode 100644 docs/snippets/idempotency/makeIdempotentBase.ts create mode 100644 docs/snippets/idempotency/makeIdempotentJmes.ts create mode 100644 docs/snippets/idempotency/makeIdempotentLambdaContext.ts delete mode 100644 docs/snippets/idempotency/testingYourCodeFunction.ts rename docs/snippets/idempotency/{dynamoDBCompositePK.ts => workingWithCompositeKey.ts} (54%) create mode 100644 docs/snippets/idempotency/workingWithCustomClient.ts create mode 100644 docs/snippets/idempotency/workingWithCustomConfig.ts create mode 100644 docs/snippets/idempotency/workingWithExceptions.ts create mode 100644 docs/snippets/idempotency/workingWithIdempotencyRequiredKey.ts create mode 100644 docs/snippets/idempotency/workingWithLocalCache.ts create mode 100644 docs/snippets/idempotency/workingWithPayloadValidation.ts create mode 100644 docs/snippets/idempotency/workingWithRecordExpiration.ts diff --git a/docs/snippets/idempotency/customizePersistenceLayer.ts b/docs/snippets/idempotency/customizePersistenceLayer.ts new file mode 100644 index 0000000000..138da79b9c --- /dev/null +++ b/docs/snippets/idempotency/customizePersistenceLayer.ts @@ -0,0 +1,35 @@ +import { makeHandlerIdempotent } from '@aws-lambda-powertools/idempotency/middleware'; +import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; +import middy from '@middy/core'; +import type { Context } from 'aws-lambda'; +import type { Request, Response } from './types'; + +const persistenceStore = new DynamoDBPersistenceLayer({ + tableName: 'idempotencyTableName', + keyAttr: 'idempotencyKey', + expiryAttr: 'expiresAt', + inProgressExpiryAttr: 'inProgressExpiresAt', + statusAttr: 'currentStatus', + dataAttr: 'resultData', + validationKeyAttr: 'validationKey', +}); + +export const handler = middy( + async (_event: Request, _context: Context): Promise => { + try { + // ... create payment + + return { + paymentId: '1234567890', + message: 'success', + statusCode: 200, + }; + } catch (error) { + throw new Error('Error creating payment'); + } + } +).use( + makeHandlerIdempotent({ + persistenceStore, + }) +); diff --git a/docs/snippets/idempotency/disablingUtility.ts b/docs/snippets/idempotency/disablingUtility.ts deleted file mode 100644 index a3eb34b60b..0000000000 --- a/docs/snippets/idempotency/disablingUtility.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { handler } from './testingYourCodeFunction'; - -describe('Function tests', () => { - const ENVIRONMENT_VARIABLES = process.env; - - beforeEach(() => { - process.env = { ...ENVIRONMENT_VARIABLES }; - }); - - afterAll(() => { - process.env = ENVIRONMENT_VARIABLES; - }); - - test('it returns the correct response', async () => { - // Prepare - process.env.POWERTOOLS_IDEMPOTENCY_DISABLED = 'true'; - - // Act - const result = await handler({}, {}); - - // Assess - expect(result).toStrictEqual({ - paymentId: 12345, - message: 'success', - statusCode: 200, - }); - }); -}); diff --git a/docs/snippets/idempotency/jestMockDynamoDB.ts b/docs/snippets/idempotency/jestMockDynamoDB.ts deleted file mode 100644 index 967ef98cba..0000000000 --- a/docs/snippets/idempotency/jestMockDynamoDB.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { handler } from './testingYourCodeFunction'; -import { - DynamoDBClient, - GetItemCommand, - ResourceNotFoundException, -} from '@aws-sdk/client-dynamodb'; -import { mockClient } from 'aws-sdk-client-mock'; -import 'aws-sdk-client-mock-jest'; - -describe('Function tests', () => { - const client = mockClient(DynamoDBClient); - - beforeEach(() => { - jest.clearAllMocks(); - }); - - afterEach(() => { - client.reset(); - }); - - test('it returns the correct response', async () => { - // Prepare - client.on(GetItemCommand).rejectsOnce( - new ResourceNotFoundException({ - $metadata: { - httpStatusCode: 404, - }, - message: 'Unable to find table', - }) - ); - - // TODO: test this test - - // Act - const result = await handler({}, {}); - - // Assess - expect(result).toStrictEqual({ message: 'Unable to find table' }); - }); -}); diff --git a/docs/snippets/idempotency/localDynamoDB.ts b/docs/snippets/idempotency/localDynamoDB.ts deleted file mode 100644 index 29e28c076f..0000000000 --- a/docs/snippets/idempotency/localDynamoDB.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { handler } from './testingYourCodeFunction'; -import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; - -describe('Function tests', () => { - test('it returns the correct response', async () => { - // Prepare - const ddbClient = new DynamoDBClient({ - endpoint: 'http://localhost:8000', - }); - - // TODO: patch the ddb client - - // Act - const result = await handler({}, {}); - - // Assess - expect(result).toStrictEqual({ - paymentId: 12345, - message: 'success', - statusCode: 200, - }); - }); -}); diff --git a/docs/snippets/idempotency/makeFunctionIdempotentBase.ts b/docs/snippets/idempotency/makeFunctionIdempotentBase.ts deleted file mode 100644 index 83d50628c1..0000000000 --- a/docs/snippets/idempotency/makeFunctionIdempotentBase.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { makeFunctionIdempotent } from '@aws-lambda-powertools/idempotency'; -import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; -import type { Context } from 'aws-lambda'; - -const persistenceStore = new DynamoDBPersistenceLayer({ - tableName: 'idempotencyTableName', -}); - -export const handler = makeFunctionIdempotent( - async (_event: unknown, _context: unknown): Promise => { - // your code goes here here - }, - { - persistenceStore, - } -); diff --git a/docs/snippets/idempotency/makeHandlerIdempotent.ts b/docs/snippets/idempotency/makeHandlerIdempotent.ts new file mode 100644 index 0000000000..3989b15cd6 --- /dev/null +++ b/docs/snippets/idempotency/makeHandlerIdempotent.ts @@ -0,0 +1,40 @@ +import { randomUUID } from 'node:crypto'; +import { makeHandlerIdempotent } from '@aws-lambda-powertools/idempotency/middleware'; +import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; +import middy from '@middy/core'; +import type { Context } from 'aws-lambda'; +import type { Request, Response, SubscriptionResult } from './types'; + +const persistenceStore = new DynamoDBPersistenceLayer({ + tableName: 'idempotencyTableName', +}); + +const createSubscriptionPayment = async ( + event: Request +): Promise => { + // ... create payment + return { + id: randomUUID(), + productId: event.productId, + }; +}; + +export const handler = middy( + async (event: Request, _context: Context): Promise => { + try { + const payment = await createSubscriptionPayment(event); + + return { + paymentId: payment.id, + message: 'success', + statusCode: 200, + }; + } catch (error) { + throw new Error('Error creating payment'); + } + } +).use( + makeHandlerIdempotent({ + persistenceStore, + }) +); diff --git a/docs/snippets/idempotency/makeIdempotentAnyFunction.ts b/docs/snippets/idempotency/makeIdempotentAnyFunction.ts new file mode 100644 index 0000000000..ead0ca408e --- /dev/null +++ b/docs/snippets/idempotency/makeIdempotentAnyFunction.ts @@ -0,0 +1,59 @@ +import { randomUUID } from 'node:crypto'; +import { + makeIdempotent, + IdempotencyConfig, +} from '@aws-lambda-powertools/idempotency'; +import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; +import type { Context } from 'aws-lambda'; +import type { Request, Response, SubscriptionResult } from './types'; + +const persistenceStore = new DynamoDBPersistenceLayer({ + tableName: 'idempotencyTableName', +}); +const config = new IdempotencyConfig({}); + +const reportSubscriptionMetrics = async ( + _transactionId: string, + _user: string +): Promise => { + // ... send notification +}; + +const createSubscriptionPayment = makeIdempotent( + async ( + transactionId: string, + event: Request + ): Promise => { + // ... create payment + return { + id: transactionId, + productId: event.productId, + }; + }, + { + persistenceStore, + dataIndexArgument: 1, + config, + } +); + +export const handler = async ( + event: Request, + context: Context +): Promise => { + config.registerLambdaContext(context); + try { + const transactionId = randomUUID(); + const payment = await createSubscriptionPayment(transactionId, event); + + await reportSubscriptionMetrics(transactionId, event.user); + + return { + paymentId: payment.id, + message: 'success', + statusCode: 200, + }; + } catch (error) { + throw new Error('Error creating payment'); + } +}; diff --git a/docs/snippets/idempotency/makeIdempotentBase.ts b/docs/snippets/idempotency/makeIdempotentBase.ts new file mode 100644 index 0000000000..857fa8dfab --- /dev/null +++ b/docs/snippets/idempotency/makeIdempotentBase.ts @@ -0,0 +1,38 @@ +import { randomUUID } from 'node:crypto'; +import { makeIdempotent } from '@aws-lambda-powertools/idempotency'; +import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; +import type { Context } from 'aws-lambda'; +import type { Request, Response, SubscriptionResult } from './types'; + +const persistenceStore = new DynamoDBPersistenceLayer({ + tableName: 'idempotencyTableName', +}); + +const createSubscriptionPayment = async ( + event: Request +): Promise => { + // ... create payment + return { + id: randomUUID(), + productId: event.productId, + }; +}; + +export const handler = makeIdempotent( + async (event: Request, _context: Context): Promise => { + try { + const payment = await createSubscriptionPayment(event); + + return { + paymentId: payment.id, + message: 'success', + statusCode: 200, + }; + } catch (error) { + throw new Error('Error creating payment'); + } + }, + { + persistenceStore, + } +); diff --git a/docs/snippets/idempotency/makeIdempotentJmes.ts b/docs/snippets/idempotency/makeIdempotentJmes.ts new file mode 100644 index 0000000000..b460f6df05 --- /dev/null +++ b/docs/snippets/idempotency/makeIdempotentJmes.ts @@ -0,0 +1,51 @@ +import { randomUUID } from 'node:crypto'; +import { + makeIdempotent, + IdempotencyConfig, +} from '@aws-lambda-powertools/idempotency'; +import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; +import type { Context } from 'aws-lambda'; +import type { Request, Response, SubscriptionResult } from './types'; + +const persistenceStore = new DynamoDBPersistenceLayer({ + tableName: 'idempotencyTableName', +}); + +const createSubscriptionPayment = async ( + user: string, + productId: string +): Promise => { + // ... create payment + return { + id: randomUUID(), + productId: productId, + }; +}; + +// Extract the idempotency key from the request headers +const config = new IdempotencyConfig({ + eventKeyJmesPath: 'headers."X-Idempotency-Key"', +}); + +export const handler = makeIdempotent( + async (event: Request, _context: Context): Promise => { + try { + const payment = await createSubscriptionPayment( + event.user, + event.productId + ); + + return { + paymentId: payment.id, + message: 'success', + statusCode: 200, + }; + } catch (error) { + throw new Error('Error creating payment'); + } + }, + { + persistenceStore, + config, + } +); diff --git a/docs/snippets/idempotency/makeIdempotentLambdaContext.ts b/docs/snippets/idempotency/makeIdempotentLambdaContext.ts new file mode 100644 index 0000000000..d90ee17eb1 --- /dev/null +++ b/docs/snippets/idempotency/makeIdempotentLambdaContext.ts @@ -0,0 +1,51 @@ +import { randomUUID } from 'node:crypto'; +import { + makeIdempotent, + IdempotencyConfig, +} from '@aws-lambda-powertools/idempotency'; +import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; +import type { Context } from 'aws-lambda'; +import type { Request, Response, SubscriptionResult } from './types'; + +const persistenceStore = new DynamoDBPersistenceLayer({ + tableName: 'idempotencyTableName', +}); +const config = new IdempotencyConfig({}); + +const createSubscriptionPayment = makeIdempotent( + async ( + transactionId: string, + event: Request + ): Promise => { + // ... create payment + return { + id: transactionId, + productId: event.productId, + }; + }, + { + persistenceStore, + dataIndexArgument: 1, + config, + } +); + +export const handler = async ( + event: Request, + context: Context +): Promise => { + // Register the Lambda context to the IdempotencyConfig instance + config.registerLambdaContext(context); + try { + const transactionId = randomUUID(); + const payment = await createSubscriptionPayment(transactionId, event); + + return { + paymentId: payment.id, + message: 'success', + statusCode: 200, + }; + } catch (error) { + throw new Error('Error creating payment'); + } +}; diff --git a/docs/snippets/idempotency/requiredIdempotencyKey.ts b/docs/snippets/idempotency/requiredIdempotencyKey.ts index 1ad36edcc6..1f6a13e286 100644 --- a/docs/snippets/idempotency/requiredIdempotencyKey.ts +++ b/docs/snippets/idempotency/requiredIdempotencyKey.ts @@ -1,6 +1,6 @@ import { IdempotencyConfig, - makeFunctionIdempotent, + makeIdempotent, } from '@aws-lambda-powertools/idempotency'; import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; @@ -14,7 +14,7 @@ const config = new IdempotencyConfig({ throwOnNoIdempotencyKey: true, }); -export const handler = makeFunctionIdempotent((_event: unknown) => ({}), { +export const handler = makeIdempotent((_event: unknown) => ({}), { persistenceStore, config, }); diff --git a/docs/snippets/idempotency/testingYourCodeFunction.ts b/docs/snippets/idempotency/testingYourCodeFunction.ts deleted file mode 100644 index c8449541a2..0000000000 --- a/docs/snippets/idempotency/testingYourCodeFunction.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { makeFunctionIdempotent } from '@aws-lambda-powertools/idempotency'; -import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; -import type { Response } from './types'; - -const persistenceStore = new DynamoDBPersistenceLayer({ - tableName: 'IdempotencyTable', -}); - -const myLambdaHandler = async ( - _event: unknown, - _context: unknown -): Promise => { - // expensive operation - return { - paymentId: 12345, - message: 'success', - statusCode: 200, - }; -}; - -export const handler = makeFunctionIdempotent(myLambdaHandler, { - persistenceStore, -}); diff --git a/docs/snippets/idempotency/types.ts b/docs/snippets/idempotency/types.ts index 2f2997eef9..42d2cd63bd 100644 --- a/docs/snippets/idempotency/types.ts +++ b/docs/snippets/idempotency/types.ts @@ -1,6 +1,5 @@ type Request = { user: string; - email: string; productId: string; }; @@ -10,6 +9,7 @@ type Response = { type SubscriptionResult = { id: string; + productId: string; }; export { Request, Response, SubscriptionResult }; diff --git a/docs/snippets/idempotency/dynamoDBCompositePK.ts b/docs/snippets/idempotency/workingWithCompositeKey.ts similarity index 54% rename from docs/snippets/idempotency/dynamoDBCompositePK.ts rename to docs/snippets/idempotency/workingWithCompositeKey.ts index 20774a78a2..8b13d122c7 100644 --- a/docs/snippets/idempotency/dynamoDBCompositePK.ts +++ b/docs/snippets/idempotency/workingWithCompositeKey.ts @@ -1,18 +1,28 @@ import { makeHandlerIdempotent } from '@aws-lambda-powertools/idempotency/middleware'; import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; import middy from '@middy/core'; +import type { Context } from 'aws-lambda'; import type { Request, Response } from './types'; const persistenceStore = new DynamoDBPersistenceLayer({ - tableName: 'IdempotencyTable', + tableName: 'idempotencyTableName', sortKeyAttr: 'sort_key', }); export const handler = middy( - async (event: Request, _context: unknown): Promise => ({ - message: 'success', - id: event.email, - }) + async (_event: Request, _context: Context): Promise => { + try { + // ... create payment + + return { + paymentId: '12345', + message: 'success', + statusCode: 200, + }; + } catch (error) { + throw new Error('Error creating payment'); + } + } ).use( makeHandlerIdempotent({ persistenceStore, diff --git a/docs/snippets/idempotency/workingWithCustomClient.ts b/docs/snippets/idempotency/workingWithCustomClient.ts new file mode 100644 index 0000000000..1577912f10 --- /dev/null +++ b/docs/snippets/idempotency/workingWithCustomClient.ts @@ -0,0 +1,32 @@ +import { makeIdempotent } from '@aws-lambda-powertools/idempotency'; +import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; +import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; +import type { Context } from 'aws-lambda'; +import type { Request, Response } from './types'; + +const customDynamoDBClient = new DynamoDBClient({ + endpoint: 'http://localhost:8000', +}); +const persistenceStore = new DynamoDBPersistenceLayer({ + tableName: 'idempotencyTableName', + awsSdkV3Client: customDynamoDBClient, +}); + +export const handler = makeIdempotent( + async (_event: Request, _context: Context): Promise => { + try { + // ... create payment + + return { + paymentId: '12345', + message: 'success', + statusCode: 200, + }; + } catch (error) { + throw new Error('Error creating payment'); + } + }, + { + persistenceStore, + } +); diff --git a/docs/snippets/idempotency/workingWithCustomConfig.ts b/docs/snippets/idempotency/workingWithCustomConfig.ts new file mode 100644 index 0000000000..6507a1a32c --- /dev/null +++ b/docs/snippets/idempotency/workingWithCustomConfig.ts @@ -0,0 +1,30 @@ +import { makeIdempotent } from '@aws-lambda-powertools/idempotency'; +import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; +import type { Context } from 'aws-lambda'; +import type { Request, Response } from './types'; + +const persistenceStore = new DynamoDBPersistenceLayer({ + tableName: 'idempotencyTableName', + clientConfig: { + region: 'us-east-1', + }, +}); + +export const handler = makeIdempotent( + async (_event: Request, _context: Context): Promise => { + try { + // ... create payment + + return { + paymentId: '12345', + message: 'success', + statusCode: 200, + }; + } catch (error) { + throw new Error('Error creating payment'); + } + }, + { + persistenceStore, + } +); diff --git a/docs/snippets/idempotency/workingWithExceptions.ts b/docs/snippets/idempotency/workingWithExceptions.ts new file mode 100644 index 0000000000..60d957d893 --- /dev/null +++ b/docs/snippets/idempotency/workingWithExceptions.ts @@ -0,0 +1,59 @@ +import { randomUUID } from 'node:crypto'; +import { + makeIdempotent, + IdempotencyConfig, +} from '@aws-lambda-powertools/idempotency'; +import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; +import type { Context } from 'aws-lambda'; +import type { Request, Response, SubscriptionResult } from './types'; + +const persistenceStore = new DynamoDBPersistenceLayer({ + tableName: 'idempotencyTableName', +}); +const config = new IdempotencyConfig({}); + +const createSubscriptionPayment = makeIdempotent( + async ( + transactionId: string, + event: Request + ): Promise => { + // ... create payment + return { + id: transactionId, + productId: event.productId, + }; + }, + { + persistenceStore, + dataIndexArgument: 1, + config, + } +); + +export const handler = async ( + event: Request, + context: Context +): Promise => { + config.registerLambdaContext(context); + /** + * If an exception is thrown before the wrapped function is called, + * no idempotency record is created. + */ + try { + const transactionId = randomUUID(); + const payment = await createSubscriptionPayment(transactionId, event); + + /** + * If an exception is thrown after the wrapped function is called, + * the idempotency record won't be affected so it's safe to retry. + */ + + return { + paymentId: payment.id, + message: 'success', + statusCode: 200, + }; + } catch (error) { + throw new Error('Error creating payment'); + } +}; diff --git a/docs/snippets/idempotency/workingWithIdempotencyRequiredKey.ts b/docs/snippets/idempotency/workingWithIdempotencyRequiredKey.ts new file mode 100644 index 0000000000..9642e6a630 --- /dev/null +++ b/docs/snippets/idempotency/workingWithIdempotencyRequiredKey.ts @@ -0,0 +1,36 @@ +import { + makeIdempotent, + IdempotencyConfig, +} from '@aws-lambda-powertools/idempotency'; +import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; +import type { Context } from 'aws-lambda'; +import type { Request, Response } from './types'; + +const persistenceStore = new DynamoDBPersistenceLayer({ + tableName: 'idempotencyTableName', +}); + +const config = new IdempotencyConfig({ + throwOnNoIdempotencyKey: true, + eventKeyJmesPath: '["user.uid", "productId"]', +}); + +export const handler = makeIdempotent( + async (_event: Request, _context: Context): Promise => { + try { + // ... create payment + + return { + paymentId: '12345', + message: 'success', + statusCode: 200, + }; + } catch (error) { + throw new Error('Error creating payment'); + } + }, + { + persistenceStore, + config, + } +); diff --git a/docs/snippets/idempotency/workingWithLocalCache.ts b/docs/snippets/idempotency/workingWithLocalCache.ts new file mode 100644 index 0000000000..8570f3f055 --- /dev/null +++ b/docs/snippets/idempotency/workingWithLocalCache.ts @@ -0,0 +1,35 @@ +import { IdempotencyConfig } from '@aws-lambda-powertools/idempotency'; +import { makeHandlerIdempotent } from '@aws-lambda-powertools/idempotency/middleware'; +import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; +import middy from '@middy/core'; +import type { Context } from 'aws-lambda'; +import type { Request, Response } from './types'; + +const persistenceStore = new DynamoDBPersistenceLayer({ + tableName: 'idempotencyTableName', +}); +const config = new IdempotencyConfig({ + useLocalCache: true, + maxLocalCacheSize: 512, +}); + +export const handler = middy( + async (_event: Request, _context: Context): Promise => { + try { + // ... create payment + + return { + paymentId: '1234567890', + message: 'success', + statusCode: 200, + }; + } catch (error) { + throw new Error('Error creating payment'); + } + } +).use( + makeHandlerIdempotent({ + persistenceStore, + config, + }) +); diff --git a/docs/snippets/idempotency/workingWithPayloadValidation.ts b/docs/snippets/idempotency/workingWithPayloadValidation.ts new file mode 100644 index 0000000000..eb582656a0 --- /dev/null +++ b/docs/snippets/idempotency/workingWithPayloadValidation.ts @@ -0,0 +1,58 @@ +import { randomUUID } from 'node:crypto'; +import { + makeIdempotent, + IdempotencyConfig, +} from '@aws-lambda-powertools/idempotency'; +import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; +import type { Context } from 'aws-lambda'; +import type { Request, Response, SubscriptionResult } from './types'; + +const persistenceStore = new DynamoDBPersistenceLayer({ + tableName: 'idempotencyTableName', +}); +const config = new IdempotencyConfig({ + eventKeyJmesPath: '["userId", "productId"]', + payloadValidationJmesPath: 'amount', +}); + +const fetchProductAmount = async (_transactionId: string): Promise => { + // ... fetch product amount + return 42; +}; + +const createSubscriptionPayment = makeIdempotent( + async (event: Request & { amount: number }): Promise => { + // ... create payment + return { + id: randomUUID(), + productId: event.productId, + }; + }, + { + persistenceStore, + dataIndexArgument: 1, + config, + } +); + +export const handler = async ( + event: Request, + context: Context +): Promise => { + config.registerLambdaContext(context); + try { + const productAmount = await fetchProductAmount(event.productId); + const payment = await createSubscriptionPayment({ + ...event, + amount: productAmount, + }); + + return { + paymentId: payment.id, + message: 'success', + statusCode: 200, + }; + } catch (error) { + throw new Error('Error creating payment'); + } +}; diff --git a/docs/snippets/idempotency/workingWithRecordExpiration.ts b/docs/snippets/idempotency/workingWithRecordExpiration.ts new file mode 100644 index 0000000000..39fa7594eb --- /dev/null +++ b/docs/snippets/idempotency/workingWithRecordExpiration.ts @@ -0,0 +1,35 @@ +import { + makeIdempotent, + IdempotencyConfig, +} from '@aws-lambda-powertools/idempotency'; +import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; +import type { Context } from 'aws-lambda'; +import type { Request, Response } from './types'; + +const persistenceStore = new DynamoDBPersistenceLayer({ + tableName: 'idempotencyTableName', +}); + +const config = new IdempotencyConfig({ + expiresAfterSeconds: 300, +}); + +export const handler = makeIdempotent( + async (_event: Request, _context: Context): Promise => { + try { + // ... create payment + + return { + paymentId: '12345', + message: 'success', + statusCode: 200, + }; + } catch (error) { + throw new Error('Error creating payment'); + } + }, + { + persistenceStore, + config, + } +); diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index cb5d36cd3c..76723a7e08 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -3,11 +3,19 @@ title: Idempotency description: Utility --- -!!! warning - This page refers to an **unreleased and upcoming utility**. Please refer to this [GitHub milestone](https://github.com/awslabs/aws-lambda-powertools-typescript/milestone/7) for the latest updates. +???+ warning + **This utility is currently released as beta developer preview** and is intended strictly for feedback and testing purposes **and not for production workloads**. The version and all future versions tagged with the `-beta` suffix should be treated as not stable. Up until before the [General Availability release](https://github.com/aws-powertools/powertools-lambda-typescript/milestone/7) we might introduce significant breaking changes and improvements in response to customers feedback. The idempotency utility provides a simple solution to convert your Lambda functions into idempotent operations which are safe to retry. +## Key features + +* Prevent Lambda handler from executing more than once on the same event payload during a time window +* Ensure Lambda handler returns the same result when called with the same payload +* Select a subset of the event as the idempotency key using JMESPath expressions +* Set a time window in which records with the same payload should be considered duplicates +* Expires in-progress executions if the Lambda function times out halfway through + ## Terminology The property of idempotency means that an operation does not cause additional side effects if it is called more than once with the same input parameters. @@ -25,10 +33,10 @@ classDiagram class IdempotencyRecord { idempotencyKey string status Status - expiryTimestamp int - inProgressExpiryTimestamp int - responseData Json~str~ - payloadHash str + expiryTimestamp number + inProgressExpiryTimestamp number + responseData Json~string~ + payloadHash string } class Status { <> @@ -42,14 +50,6 @@ classDiagram Idempotency record representation -## Key features - -* Prevent Lambda handler from executing more than once on the same event payload during a time window -* Ensure Lambda handler returns the same result when called with the same payload -* Select a subset of the event as the idempotency key using JMESPath expressions -* Set a time window in which records with the same payload should be considered duplicates -* Expires in-progress executions if the Lambda function times out halfway through - ## Getting started ### IAM Permissions @@ -57,7 +57,7 @@ classDiagram Your Lambda function IAM Role must have `dynamodb:GetItem`, `dynamodb:PutItem`, `dynamodb:UpdateItem` and `dynamodb:DeleteItem` IAM permissions before using this feature. ???+ note - If you're using one of our [examples below](#required-resources) the required permissions are already included. + If you're using one of our examples: [AWS Serverless Application Model (SAM)](#required-resources), [AWS Cloud Development Kit (CDK)](#required-resources), or [Terraform](#required-resources) the required permissions are already included. ### Required resources @@ -74,11 +74,10 @@ If you're not [changing the default configuration for the DynamoDB persistence l | Partition key | `id` | | TTL attribute name | `expiration` | This can only be configured after your table is created if you're using AWS Console | - ???+ tip "Tip: You can share a single state table for all functions" - You can reuse the same DynamoDB table to store idempotency state. We add `module_name` and [qualified name for classes and functions](https://peps.python.org/pep-3155/) in addition to the idempotency key as a hash key. + You can reuse the same DynamoDB table to store idempotency state. We add the Lambda function name in addition to the idempotency key as a hash key. - + === "AWS Serverless Application Model (SAM) example" ```yaml hl_lines="6-14 24-31" @@ -114,11 +113,6 @@ If you're not [changing the default configuration for the DynamoDB persistence l - dynamodb:DeleteItem Resource: !GetAtt IdempotencyTable.Arn ``` -=== "AWS Cloud Development Kit (CDK)" - - ```typescript hl_lines="10 13 16 19-21" - import { Table, Attribute } from 'aws-cdk-lib/aws-dynamodb'; - ``` === "Terraform" @@ -153,8 +147,8 @@ If you're not [changing the default configuration for the DynamoDB persistence l resource "aws_lambda_function" "IdempotencyFunction" { function_name = "IdempotencyFunction" role = aws_iam_role.IdempotencyFunctionRole.arn - runtime = "python3.10" - handler = "app.lambda_handler" + runtime = "nodejs18.x" + handler = "index.handler" filename = "lambda.zip" } @@ -203,132 +197,84 @@ If you're not [changing the default configuration for the DynamoDB persistence l } ``` -??? warning "Warning: Large responses with DynamoDB persistence layer" - When using this utility with DynamoDB, your function's responses must be [smaller than 400KB](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Limits.html#limits-items). +???+ warning "Warning: Large responses with DynamoDB persistence layer" + When using this utility with DynamoDB, your function's responses must be [smaller than 400KB](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Limits.html#limits-items){target="_blank"}. Larger items cannot be written to DynamoDB and will cause exceptions. ???+ info "Info: DynamoDB" Each function invocation will generally make 2 requests to DynamoDB. If the - result returned by your Lambda is less than 1kb, you can expect 2 WCUs per invocation. For retried invocations, you will see 1WCU and 1RCU. Review the [DynamoDB pricing documentation](https://aws.amazon.com/dynamodb/pricing/) to + result returned by your Lambda is less than 1kb, you can expect 2 WCUs per invocation. For retried invocations, you will + see 1WCU and 1RCU. Review the [DynamoDB pricing documentation](https://aws.amazon.com/dynamodb/pricing/){target="_blank"} to estimate the cost. -### MakeFunctionIdempotent function wrapper +### MakeIdempotent function wrapper -You can quickly start by initializing the `DynamoDBPersistenceLayer` class and using it with the `makeFunctionIdempotent` function wrapper on your Lambda handler. +You can quickly start by initializing the `DynamoDBPersistenceLayer` class and using it with the `makeIdempotent` function wrapper on your Lambda handler. ???+ note - In this example, the entire Lambda handler is treated as a single idempotent operation. If your Lambda handler can cause multiple side effects, or you're only interested in making a specific logic idempotent, use the `makeFunctionIdempotent` high-order function only on the specific logic you want to make idempotent instead. + In this example, the entire Lambda handler is treated as a single idempotent operation. If your Lambda handler can cause multiple side effects, or you're only interested in making a specific logic idempotent, use the `makeIdempotent` high-order function only on the function that needs to be idempotent. !!! tip "See [Choosing a payload subset for idempotency](#choosing-a-payload-subset-for-idempotency) for more elaborate use cases." === "index.ts" - ```typescript hl_lines="1-2 6-8 12-14 23" - --8<-- "docs/snippets/idempotency/makeFunctionIdempotentBase.ts" + ```typescript hl_lines="2-3 21 35-38" + --8<-- "docs/snippets/idempotency/makeIdempotentBase.ts" ``` -=== "Example event" +=== "Types" - ```json - { - "username": "xyz", - "productId": "123456789" - } + ```typescript + --8<-- "docs/snippets/idempotency/types.ts::13" ``` After processing this request successfully, a second request containing the exact same payload above will now return the same response, ensuring our customer isn't charged twice. !!! question "New to idempotency concept? Please review our [Terminology](#terminology) section if you haven't yet." -### IdempotentMethod decorator - -Similar to [idempotent decorator](#idempotent-decorator), you can use `idempotent_function` decorator for any synchronous Python function. - -When using `idempotent_function`, you must tell us which keyword parameter in your function signature has the data we should use via **`data_keyword_argument`**. - -!!! tip "We support JSON serializable data, [Python Dataclasses](https://docs.python.org/3.7/library/dataclasses.html){target="_blank"}, [Parser/Pydantic Models](parser.md){target="_blank"}, and our [Event Source Data Classes](./data_classes.md){target="_blank"}." +You can also use the `makeIdempotent` function wrapper on any function that returns a response to make it idempotent. This is useful when you want to make a specific logic idempotent, for example when your Lambda handler performs multiple side effects and you only want to make a specific one idempotent. ???+ warning "Limitation" - Make sure to call your decorated function using keyword arguments. - -=== "dataclass_sample.py" - - ```python hl_lines="3-4 23 33" - from dataclasses import dataclass - - from aws_lambda_powertools.utilities.idempotency import ( - DynamoDBPersistenceLayer, IdempotencyConfig, idempotent_function) + Make sure to return a JSON serializable response from your function, otherwise you'll get an error. - dynamodb = DynamoDBPersistenceLayer(table_name="idem") - config = IdempotencyConfig( - event_key_jmespath="order_id", # see Choosing a payload subset section - use_local_cache=True, - ) +When using `makeIdempotent` on arbitrary functions, you can tell us which argument in your function signature has the data we should use via **`dataIndexArgument`**. If you don't specify this argument, we'll use the first argument in the function signature. - @dataclass - class OrderItem: - sku: str - description: str - - @dataclass - class Order: - item: OrderItem - order_id: int - - - @idempotent_function(data_keyword_argument="order", config=config, persistence_store=dynamodb) - def process_order(order: Order): - return f"processed order {order.order_id}" +???+ note + The function in the example below has two arguments, note that while wrapping it with the `makeIdempotent` high-order function, we specify the `dataIndexArgument` as `1` to tell the decorator that the second argument is the one that contains the data we should use to make the function idempotent. Remember that arguments are zero-indexed, so the first argument is `0`, the second is `1`, and so on. - def lambda_handler(event, context): - config.register_lambda_context(context) # see Lambda timeouts section - order_item = OrderItem(sku="fake", description="sample") - order = Order(item=order_item, order_id="fake-id") +=== "index.ts" - # `order` parameter must be called as a keyword argument to work - process_order(order=order) + ```typescript hl_lines="22 34-38" + --8<-- "docs/snippets/idempotency/makeIdempotentAnyFunction.ts" ``` -=== "parser_pydantic_sample.py" - - ```python hl_lines="1-2 22 32" - from aws_lambda_powertools.utilities.idempotency import ( - DynamoDBPersistenceLayer, IdempotencyConfig, idempotent_function) - from aws_lambda_powertools.utilities.parser import BaseModel - - dynamodb = DynamoDBPersistenceLayer(table_name="idem") - config = IdempotencyConfig( - event_key_jmespath="order_id", # see Choosing a payload subset section - use_local_cache=True, - ) +=== "Types" + ```typescript + --8<-- "docs/snippets/idempotency/types.ts::13" + ``` - class OrderItem(BaseModel): - sku: str - description: str +### MakeHandlerIdempotent Middy middleware +!!! tip "A note about Middy" + Currently we support only Middy `v3.x` that you can install it by running `npm i @middy/core@~3`. + Check their docs to learn more about [Middy and its middleware stack](https://middy.js.org/docs/intro/getting-started){target="_blank"} as well as [best practices when working with Powertools](https://middy.js.org/docs/integrations/lambda-powertools#best-practices){target="_blank"}. - class Order(BaseModel): - item: OrderItem - order_id: int +If you are using [Middy](https://middy.js.org){target="_blank"} as your middleware engine, you can use the `makeHandlerIdempotent` middleware to make your Lambda handler idempotent. Similar to the `makeIdempotent` function wrapper, you can quickly make your Lambda handler idempotent by initializing the `DynamoDBPersistenceLayer` class and using it with the `makeHandlerIdempotent` middleware. +=== "index.ts" - @idempotent_function(data_keyword_argument="order", config=config, persistence_store=dynamodb) - def process_order(order: Order): - return f"processed order {order.order_id}" + ```typescript hl_lines="18 29-33" + --8<-- "docs/snippets/idempotency/makeHandlerIdempotent.ts" + ``` - def lambda_handler(event, context): - config.register_lambda_context(context) # see Lambda timeouts section - order_item = OrderItem(sku="fake", description="sample") - order = Order(item=order_item, order_id="fake-id") +=== "Types" - # `order` parameter must be called as a keyword argument to work - process_order(order=order) + ```typescript + --8<-- "docs/snippets/idempotency/types.ts::13" ``` -### makeFunctionIdempotent higher-order function - ### Choosing a payload subset for idempotency ???+ tip "Tip: Dealing with always changing payloads" @@ -347,35 +293,10 @@ Imagine the function executes successfully, but the client never receives the re ???+ tip "Deserializing JSON strings in payloads for increased accuracy." The payload extracted by the `eventKeyJmesPath` is treated as a string by default. This means there could be differences in whitespace even when the JSON payload itself is identical. - To alter this behaviour, we can use the [JMESPath built-in function](jmespath_functions.md#powertools_json-function) `powertools_json()` to treat the payload as a JSON object (dict) rather than a string. - -=== "payment.py" - - ```python hl_lines="2-4 10 12 15 20" - import json - from aws_lambda_powertools.utilities.idempotency import ( - IdempotencyConfig, DynamoDBPersistenceLayer, idempotent - ) - - persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") - - # Deserialize JSON string under the "body" key - # then extract "user" and "product_id" data - config = IdempotencyConfig(event_key_jmespath="powertools_json(body).[user, product_id]") - - @idempotent(config=config, persistence_store=persistence_layer) - def handler(event, context): - body = json.loads(event['body']) - payment = create_subscription_payment( - user=body['user'], - product=body['product_id'] - ) - ... - return { - "payment_id": payment.id, - "message": "success", - "statusCode": 200 - } +=== "index.ts" + + ```typescript hl_lines="4 26-28 49" + --8<-- "docs/snippets/idempotency/makeIdempotentJmes.ts" ``` === "Example event" @@ -388,7 +309,7 @@ Imagine the function executes successfully, but the client never receives the re "rawQueryString":"", "headers": { "Header1": "value1", - "Header2": "value2" + "X-Idempotency-Key": "abcdefg" }, "requestContext":{ "accountId":"123456789012", @@ -408,17 +329,23 @@ Imagine the function executes successfully, but the client never receives the re "time":"10/Feb/2021:13:40:43 +0000", "timeEpoch":1612964443723 }, - "body":"{\"user\":\"xyz\",\"product_id\":\"123456789\"}", + "body":"{\"user\":\"xyz\",\"productId\":\"123456789\"}", "isBase64Encoded":false } ``` +=== "Types" + + ```typescript + --8<-- "docs/snippets/idempotency/types.ts::13" + ``` + ### Lambda timeouts ???+ note - This is automatically done when you decorate your Lambda handler with [@idempotent decorator](#idempotent-decorator). + This is automatically done when you wrap your Lambda handler with the [makeIdempotent](#makeIdempotent-function-wrapper) function wrapper, or use the [`makeHandlerIdempotent`](#makeHandlerIdempotent-middy-middleware) Middy middleware. -To prevent against extended failed retries when a [Lambda function times out](https://aws.amazon.com/premiumsupport/knowledge-center/lambda-verify-invocation-timeouts/), Powertools for AWS Lambda (Python) calculates and includes the remaining invocation available time as part of the idempotency record. +To prevent against extended failed retries when a [Lambda function times out](https://aws.amazon.com/premiumsupport/knowledge-center/lambda-verify-invocation-timeouts/), Powertools for AWS calculates and includes the remaining invocation available time as part of the idempotency record. ???+ example If a second invocation happens **after** this timestamp, and the record is marked as `INPROGRESS`, we will execute the invocation again as if it was in the `EXPIRED` state (e.g, `expire_seconds` field elapsed). @@ -426,34 +353,19 @@ To prevent against extended failed retries when a [Lambda function times out](ht This means that if an invocation expired during execution, it will be quickly executed again on the next retry. ???+ important - If you are only using the [@idempotent_function decorator](#idempotent_function-decorator) to guard isolated parts of your code, you must use `register_lambda_context` available in the [idempotency config object](#customizing-the-default-behavior) to benefit from this protection. + If you are only using the [makeIdempotent function wrapper](#makeIdempotent-function-wrapper) to guard isolated parts of your code, you must use `registerLambdaContext` available in the [idempotency config object](#customizing-the-default-behavior) to benefit from this protection. Here is an example on how you register the Lambda context in your handler: -```python hl_lines="8 16" title="Registering the Lambda context" -from aws_lambda_powertools.utilities.data_classes.sqs_event import SQSRecord -from aws_lambda_powertools.utilities.idempotency import ( - IdempotencyConfig, idempotent_function -) - -persistence_layer = DynamoDBPersistenceLayer(table_name="...") - -config = IdempotencyConfig() - -@idempotent_function(data_keyword_argument="record", persistence_store=persistence_layer, config=config) -def record_handler(record: SQSRecord): - return {"message": record["body"]} - +=== "Registering Lambda Context" -def lambda_handler(event, context): - config.register_lambda_context(context) - - return record_handler(event) -``` + ```typescript hl_lines="13 38" + --8<-- "docs/snippets/idempotency/makeIdempotentLambdaContext.ts" + ``` ### Handling exceptions -If you are using the `idempotent` decorator on your Lambda handler, any unhandled exceptions that are raised during the code execution will cause **the record in the persistence layer to be deleted**. +If you are making on your entire Lambda handler idempotent, any unhandled exceptions that are raised during the code execution will cause **the record in the persistence layer to be deleted**. This means that new invocations will execute your code again despite having the same payload. If you don't want the record to be deleted, you need to catch exceptions within the idempotent function and return a successful response.
@@ -474,33 +386,20 @@ sequenceDiagram Idempotent sequence exception
-If you are using `idempotent_function`, any unhandled exceptions that are raised _inside_ the decorated function will cause the record in the persistence layer to be deleted, and allow the function to be executed again if retried. - -If an Exception is raised _outside_ the scope of the decorated function and after your function has been called, the persistent record will not be affected. In this case, idempotency will be maintained for your decorated function. Example: +If you are using `makeIdempotent` on any other function, any unhandled exceptions that are thrown _inside_ the wrapped function will cause the record in the persistence layer to be deleted, and allow the function to be executed again if retried. -```python hl_lines="2-4 8-10" title="Exception not affecting idempotency record sample" -def lambda_handler(event, context): - # If an exception is raised here, no idempotent record will ever get created as the - # idempotent function does not get called - do_some_stuff() +If an error is thrown _outside_ the scope of the decorated function and after your function has been called, the persistent record will not be affected. In this case, idempotency will be maintained for your decorated function. Example: - result = call_external_service(data={"user": "user1", "id": 5}) +=== "Handling exceptions" - # This exception will not cause the idempotent record to be deleted, since it - # happens after the decorated function has been successfully called - raise Exception - - -@idempotent_function(data_keyword_argument="data", config=config, persistence_store=dynamodb) -def call_external_service(data: dict, **kwargs): - result = requests.post('http://example.com', json={"user": data['user'], "transaction_id": data['id']} - return result.json() -``` + ```typescript hl_lines="18-22 28 31" + --8<-- "docs/snippets/idempotency/workingWithExceptions.ts" + ``` ???+ warning - **We will raise `IdempotencyPersistenceLayerError`** if any of the calls to the persistence layer fail unexpectedly. + **We will throw `IdempotencyPersistenceLayerError`** if any of the calls to the persistence layer fail unexpectedly. - As this happens outside the scope of your decorated function, you are not able to catch it if you're using the `idempotent` decorator on your Lambda handler. + As this happens outside the scope of your decorated function, you are not able to catch it when making your Lambda handler idempotent. ### Idempotency request flow @@ -666,6 +565,36 @@ sequenceDiagram Idempotent request during and after Lambda timeouts +#### Optional idempotency key + +
+```mermaid +sequenceDiagram + participant Client + participant Lambda + participant Persistence Layer + alt request with idempotency key + Client->>Lambda: Invoke (event) + Lambda->>Persistence Layer: Get or set idempotency_key=hash(payload) + activate Persistence Layer + Note over Lambda,Persistence Layer: Set record status to INPROGRESS.
Prevents concurrent invocations
with the same payload + Lambda-->>Lambda: Call your function + Lambda->>Persistence Layer: Update record with result + deactivate Persistence Layer + Persistence Layer-->>Persistence Layer: Update record + Note over Lambda,Persistence Layer: Set record status to COMPLETE.
New invocations with the same payload
now return the same result + Lambda-->>Client: Response sent to client + else request(s) without idempotency key + Client->>Lambda: Invoke (event) + Note over Lambda: Idempotency key is missing + Note over Persistence Layer: Skips any operation to fetch, update, and delete + Lambda-->>Lambda: Call your function + Lambda-->>Client: Response sent to client + end +``` +Optional idempotency key +
+ ## Advanced ### Persistence layers @@ -674,51 +603,43 @@ sequenceDiagram This persistence layer is built-in, and you can either use an existing DynamoDB table or create a new one dedicated for idempotency state (recommended). -```python hl_lines="5-10" title="Customizing DynamoDBPersistenceLayer to suit your table structure" -from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer - -persistence_layer = DynamoDBPersistenceLayer( - table_name="IdempotencyTable", - key_attr="idempotency_key", - expiry_attr="expires_at", - in_progress_expiry_attr="in_progress_expires_at", - status_attr="current_status", - data_attr="result_data", - validation_key_attr="validation_key", -) -``` +=== "Customizing DynamoDBPersistenceLayer to suit your table structure" + + ```typescript hl_lines="7-15" + --8<-- "docs/snippets/idempotency/workingWithExceptions.ts" + ``` When using DynamoDB as a persistence layer, you can alter the attribute names by passing these parameters when initializing the persistence layer: -| Parameter | Required | Default | Description | -| --------------------------- | ------------------ | ------------------------------------ | -------------------------------------------------------------------------------------------------------- | -| **table_name** | :heavy_check_mark: | | Table name to store state | -| **key_attr** | | `id` | Partition key of the table. Hashed representation of the payload (unless **sort_key_attr** is specified) | -| **expiry_attr** | | `expiration` | Unix timestamp of when record expires | -| **in_progress_expiry_attr** | | `in_progress_expiration` | Unix timestamp of when record expires while in progress (in case of the invocation times out) | -| **status_attr** | | `status` | Stores status of the lambda execution during and after invocation | -| **data_attr** | | `data` | Stores results of successfully executed Lambda handlers | -| **validation_key_attr** | | `validation` | Hashed representation of the parts of the event used for validation | -| **sort_key_attr** | | | Sort key of the table (if table is configured with a sort key). | -| **static_pk_value** | | `idempotency#{LAMBDA_FUNCTION_NAME}` | Static value to use as the partition key. Only used when **sort_key_attr** is set. | +| Parameter | Required | Default | Description | +| ------------------------ | ------------------ | ------------------------------------ | -------------------------------------------------------------------------------------------------------- | +| **tableName** | :heavy_check_mark: | | Table name to store state | +| **keyAttr** | | `id` | Partition key of the table. Hashed representation of the payload (unless **sort_key_attr** is specified) | +| **expiryAttr** | | `expiration` | Unix timestamp of when record expires | +| **inProgressExpiryAttr** | | `in_progress_expiration` | Unix timestamp of when record expires while in progress (in case of the invocation times out) | +| **statusAttr** | | `status` | Stores status of the lambda execution during and after invocation | +| **dataAttr** | | `data` | Stores results of successfully executed Lambda handlers | +| **validationKeyAttr** | | `validation` | Hashed representation of the parts of the event used for validation | +| **sortKeyAttr** | | | Sort key of the table (if table is configured with a sort key). | +| **staticPkValue** | | `idempotency#{LAMBDA_FUNCTION_NAME}` | Static value to use as the partition key. Only used when **sort_key_attr** is set. | ### Customizing the default behavior -Idempotent decorator can be further configured with **`IdempotencyConfig`** as seen in the previous example. These are the available options for further configuration +Idempotent decorator can be further configured with **`IdempotencyConfig`** as seen in the previous examples. These are the available options for further configuration -| Parameter | Default | Description | -| ------------------------------- | ------- | ----------------------------------------------------------------------------------------------------------------------------------------- | -| **event_key_jmespath** | `""` | JMESPath expression to extract the idempotency key from the event record using [built-in functions](/utilities/jmespath_functions) | -| **payload_validation_jmespath** | `""` | JMESPath expression to validate whether certain parameters have changed in the event while the event payload | -| **raise_on_no_idempotency_key** | `False` | Raise exception if no idempotency key was found in the request | -| **expires_after_seconds** | 3600 | The number of seconds to wait before a record is expired | -| **use_local_cache** | `False` | Whether to locally cache idempotency results | -| **local_cache_max_items** | 256 | Max number of items to store in local cache | -| **hash_function** | `md5` | Function to use for calculating hashes, as provided by [hashlib](https://docs.python.org/3/library/hashlib.html) in the standard library. | +| Parameter | Default | Description | +| ----------------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **eventKeyJmespath** | `''` | JMESPath expression to extract the idempotency key from the event | +| **payloadValidationJmespath** | `''` | JMESPath expression to validate whether certain parameters have changed in the event while the event payload | +| **throwOnNoIdempotencyKey** | `false` | Throw an error if no idempotency key was found in the request | +| **expiresAfterSeconds** | 3600 | The number of seconds to wait before a record is expired | +| **useLocalCache** | `false` | Whether to locally cache idempotency results | +| **localCacheMaxItems** | 256 | Max number of items to store in local cache | +| **hashFunction** | `md5` | Function to use for calculating hashes, as provided by the [crypto](https://nodejs.org/api/crypto.html#cryptocreatehashalgorithm-options){target="_blank"} module in the standard library. | ### Handling concurrent executions with the same payload -This utility will raise an **`IdempotencyAlreadyInProgressError`** exception if you receive **multiple invocations with the same payload while the first invocation hasn't completed yet**. +This utility will throw an **`IdempotencyAlreadyInProgressError`** error if you receive **multiple invocations with the same payload while the first invocation hasn't completed yet**. ???+ info If you receive `IdempotencyAlreadyInProgressError`, you can safely retry the operation. @@ -732,25 +653,15 @@ This is a locking mechanism for correctness. Since we don't know the result from ???+ note "Note: This in-memory cache is local to each Lambda execution environment" This means it will be effective in cases where your function's concurrency is low in comparison to the number of "retry" invocations with the same payload, because cache might be empty. -You can enable in-memory caching with the **`use_local_cache`** parameter: +You can enable in-memory caching with the **`useLocalCache`** parameter: -```python hl_lines="8 11" title="Caching idempotent transactions in-memory to prevent multiple calls to storage" -from aws_lambda_powertools.utilities.idempotency import ( - IdempotencyConfig, DynamoDBPersistenceLayer, idempotent -) +=== "Caching idempotent transactions in-memory to prevent multiple calls to storage" -persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") -config = IdempotencyConfig( - event_key_jmespath="body", - use_local_cache=True, -) - -@idempotent(config=config, persistence_store=persistence_layer) -def handler(event, context): - ... -``` + ```typescript hl_lines="12-13" + --8<-- "docs/snippets/idempotency/workingWithExceptions.ts" + ``` -When enabled, the default is to cache a maximum of 256 records in each Lambda execution environment - You can change it with the **`local_cache_max_items`** parameter. +When enabled, the default is to cache a maximum of 256 records in each Lambda execution environment - You can change it with the **`maxLocalCacheSize`** parameter. ### Expiring idempotency records @@ -758,23 +669,13 @@ When enabled, the default is to cache a maximum of 256 records in each Lambda ex In most cases, it is not desirable to store the idempotency records forever. Rather, you want to guarantee that the same payload won't be executed within a period of time. -You can change this window with the **`expires_after_seconds`** parameter: - -```python hl_lines="8 11" title="Adjusting idempotency record expiration" -from aws_lambda_powertools.utilities.idempotency import ( - IdempotencyConfig, DynamoDBPersistenceLayer, idempotent -) +You can change this window with the **`expiresAfterSeconds`** parameter: -persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") -config = IdempotencyConfig( - event_key_jmespath="body", - expires_after_seconds=5*60, # 5 minutes -) +=== "Adjusting idempotency record expiration" -@idempotent(config=config, persistence_store=persistence_layer) -def handler(event, context): - ... -``` + ```typescript hl_lines="14" + --8<-- "docs/snippets/idempotency/workingWithRecordExpiration.ts" + ``` This will mark any records older than 5 minutes as expired, and [your function will be executed as normal if it is invoked with a matching payload](#expired-idempotency-records). @@ -796,86 +697,36 @@ This will mark any records older than 5 minutes as expired, and [your function w By default, we will return the same result as it returned before, however in this instance it may be misleading; we provide a fail fast payload validation to address this edge case. -With **`payload_validation_jmespath`**, you can provide an additional JMESPath expression to specify which part of the event body should be validated against previous idempotent invocations - -=== "app.py" - - ```python hl_lines="7 11 18 25" - from aws_lambda_powertools.utilities.idempotency import ( - IdempotencyConfig, DynamoDBPersistenceLayer, idempotent - ) - - config = IdempotencyConfig( - event_key_jmespath="[userDetail, productId]", - payload_validation_jmespath="amount" - ) - persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") - - @idempotent(config=config, persistence_store=persistence_layer) - def handler(event, context): - # Creating a subscription payment is a side - # effect of calling this function! - payment = create_subscription_payment( - user=event['userDetail']['username'], - product=event['product_id'], - amount=event['amount'] - ) - ... - return { - "message": "success", - "statusCode": 200, - "payment_id": payment.id, - "amount": payment.amount - } - ``` +With **`payloadValidationJmesPath`**, you can provide an additional JMESPath expression to specify which part of the event body should be validated against previous idempotent invocations -=== "Example Event 1" +=== "Payload validation" - ```json hl_lines="8" - { - "userDetail": { - "username": "User1", - "user_email": "user@example.com" - }, - "productId": 1500, - "charge_type": "subscription", - "amount": 500 - } + ```typescript hl_lines="14-15" + --8<-- "docs/snippets/idempotency/workingWithPayloadValidation.ts" ``` -=== "Example Event 2" - - ```json hl_lines="8" - { - "userDetail": { - "username": "User1", - "user_email": "user@example.com" - }, - "productId": 1500, - "charge_type": "subscription", - "amount": 1 - } - ``` - -In this example, the **`userDetail`** and **`productId`** keys are used as the payload to generate the idempotency key, as per **`event_key_jmespath`** parameter. +In this example, the **`userId`** and **`productId`** keys are used as the payload to generate the idempotency key, as per **`eventKeyJmespath`** parameter. ???+ note If we try to send the same request but with a different amount, we will raise **`IdempotencyValidationError`**. Without payload validation, we would have returned the same result as we did for the initial request. Since we're also returning an amount in the response, this could be quite confusing for the client. -By using **`payload_validation_jmespath="amount"`**, we prevent this potentially confusing behavior and instead raise an Exception. +By using **`payloadValidationJmesPath="amount"`**, we prevent this potentially confusing behavior and instead throw an error. ### Making idempotency key required -{ -const If you new want to enforce that an idempotency key is required, you can set **`throwOnNoIdempotencyKey: t* to `True`.} -; -This means that we will raise **`IdempotencyKeyError`** if the evaluation of **`event_key_jmespath`** is `None`. -=== "app.py" +If you want to enforce that an idempotency key is required, you can set **`throwOnNoIdempotencyKey`** to `true`. + +This means that we will raise **`IdempotencyKeyError`** if the evaluation of **`eventKeyJmesPath`** results in an empty subset. + +???+ warning + To prevent errors, transactions will not be treated as idempotent if **`throwOnNoIdempotencyKey`** is set to `false` and the evaluation of **`eventKeyJmesPath`** is an empty result. Therefore, no data will be fetched, stored, or deleted in the idempotency storage layer. + +=== "Idempotency key required" - ```typescript hl_lines="9-10 13" - + ```typescript hl_lines="14-15" + --8<-- "docs/snippets/idempotency/workingWithIdempotencyRequiredKey.ts" ``` === "Success Event" @@ -884,147 +735,63 @@ This means that we will raise **`IdempotencyKeyError`** if the evaluation of **` { "user": { "uid": "BB0D045C-8878-40C8-889E-38B3CB0A61B1", - "name": "foo bar" + "name": "Foo" }, - "orderId": 10000 + "productId": 10000 } ``` === "Failure Event" - Notice that `orderId` is now accidentally within the `user` key - ```json hl_lines="3 5" { "user": { "uid": "BB0D045C-8878-40C8-889E-38B3CB0A61B1", - "name": "foo bar", - "orderId": 10000 - }, + "name": "foo", + "productId": 10000 + } } ``` ### Customizing boto configuration -The **`boto_config`** and **`boto3_session`** parameters enable you to pass in a custom [botocore config object](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html) or a custom [boto3 session](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html) when constructing the persistence store. - -=== "Custom session" +The **`clientConfig`** and **`awsSdkV3Client`** parameters enable you to pass in custom configurations or your own [DynamoDBClient](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-dynamodb/classes/dynamodbclient.html){target="_blank"} when constructing the persistence store. - ```python hl_lines="1 6 9 14" - import boto3 - from aws_lambda_powertools.utilities.idempotency import ( - IdempotencyConfig, DynamoDBPersistenceLayer, idempotent - ) +=== "Passing specific configuration" - boto3_session = boto3.session.Session() - persistence_layer = DynamoDBPersistenceLayer( - table_name="IdempotencyTable", - boto3_session=boto3_session - ) + ```typescript hl_lines="8-10" + --8<-- "docs/snippets/idempotency/workingWithCustomConfig.ts" + ``` - config = IdempotencyConfig(event_key_jmespath="body") +=== "Passing custom DynamoDBClient" - @idempotent(config=config, persistence_store=persistence_layer) - def handler(event, context): - ... - ``` -=== "Custom config" - - ```python hl_lines="1 7 10" - from botocore.config import Config - from aws_lambda_powertools.utilities.idempotency import ( - IdempotencyConfig, DynamoDBPersistenceLayer, idempotent - ) - - config = IdempotencyConfig(event_key_jmespath="body") - boto_config = Config() - persistence_layer = DynamoDBPersistenceLayer( - table_name="IdempotencyTable", - boto_config=boto_config - ) - - @idempotent(config=config, persistence_store=persistence_layer) - def handler(event, context): - ... + ```typescript hl_lines="7-9 12" + --8<-- "docs/snippets/idempotency/workingWithCustomClient.ts" ``` ### Using a DynamoDB table with a composite primary key -When using a composite primary key table (hash+range key), use the `sortKeyAttr` parameter when initializing your persistence layer. +When using a composite primary key table (hash+range key), use `sortKeyAttr` parameter when initializing your persistence layer. With this setting, we will save the idempotency key in the sort key instead of the primary key. By default, the primary key will now be set to `idempotency#{LAMBDA_FUNCTION_NAME}`. You can optionally set a static value for the partition key using the `staticPkValue` parameter. -```python hl_lines="5" title="Reusing a DynamoDB table that uses a composite primary key" ---8<-- "docs/snippets/idempotency/dynamoDBCompositePK.ts" -``` - - - -The example function above would cause data to be stored in DynamoDB like this: - -| id | sort_key | expiration | status | data | -| ---------------------------- | -------------------------------- | ---------- | ----------- | ------------------------------------ | -| idempotency#MyLambdaFunction | 1e956ef7da78d0cb890be999aecc0c9e | 1636549553 | COMPLETED | {"id": 12391, "message": "success"} | -| idempotency#MyLambdaFunction | 2b2cdb5f86361e97b4383087c1ffdf27 | 1636549571 | COMPLETED | {"id": 527212, "message": "success"} | -| idempotency#MyLambdaFunction | f091d2527ad1c78f05d54cc3f363be80 | 1636549585 | IN_PROGRESS | | - - - -## Testing your code - -The idempotency utility provides several routes to test your code. +=== "Reusing a DynamoDB table that uses a composite primary key" -### Disabling the idempotency utility - -When testing your code, you may wish to disable the idempotency logic altogether and focus on testing your business logic. To do this, you can set the environment variable `POWERTOOLS_IDEMPOTENCY_DISABLED` with a truthy value. - -=== "index.test.ts" - - ```python hl_lines="16" - --8<-- "docs/snippets/idempotency/disablingUtility.ts" + ```typescript hl_lines="14-15" + --8<-- "docs/snippets/idempotency/workingWithCompositeKey.ts" ``` -=== "index.ts" - - ```typescript - --8<-- "docs/snippets/idempotency/testingYourCodeFunction.ts" - ``` - -### Testing with DynamoDB Local - -To test with [DynamoDB Local](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocal.DownloadingAndRunning.html), you can replace the `DynamoDBClient` used by the persistence layer with one you create inside your tests. This allows you to set a local endpoint. - -=== "index.test.ts" - - ```python - --8<-- "docs/snippets/idempotency/localDynamoDB.ts" - ``` - -=== "index.ts" - - ```typescript - --8<-- "docs/snippets/idempotency/testingYourCodeFunction.ts" - ``` - -### How do I mock all DynamoDB I/O operations - -The idempotency utility lazily creates an AWS SDK for JavaScript v3 [`DynamoDBClient`](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-dynamodb/index.html) which it uses to access DynamoDB. This means it is possible to mock or spy on the API calls. - -=== "index.test.ts" - - ```python - --8<-- "docs/snippets/idempotency/jestMockDynamoDB.ts" - ``` - -=== "index.ts" +The example function above would cause data to be stored in DynamoDB like this: - ```typescript - --8<-- "docs/snippets/idempotency/testingYourCodeFunction.ts" - ``` +| id | sort_key | expiration | status | data | +| ---------------------------- | -------------------------------- | ---------- | ----------- | ---------------------------------------------------------------- | +| idempotency#MyLambdaFunction | 1e956ef7da78d0cb890be999aecc0c9e | 1636549553 | COMPLETED | {"paymentId": "12345, "message": "success", "statusCode": 200} | +| idempotency#MyLambdaFunction | 2b2cdb5f86361e97b4383087c1ffdf27 | 1636549571 | COMPLETED | {"paymentId": "527212", "message": "success", "statusCode": 200} | +| idempotency#MyLambdaFunction | f091d2527ad1c78f05d54cc3f363be80 | 1636549585 | IN_PROGRESS | | ## Extra resources If you're interested in a deep dive on how Amazon uses idempotency when building our APIs, check out -[this article](https://aws.amazon.com/builders-library/making-retries-safe-with-idempotent-APIs/). \ No newline at end of file +[this article](https://aws.amazon.com/builders-library/making-retries-safe-with-idempotent-APIs/){target="_blank"}. \ No newline at end of file From 515e74359dcac488b0b30006d07a60e0f90dab66 Mon Sep 17 00:00:00 2001 From: Andrea Amorosi Date: Mon, 10 Jul 2023 13:39:27 +0200 Subject: [PATCH 6/7] highlights --- docs/utilities/idempotency.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 76723a7e08..6078cb7978 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -265,7 +265,7 @@ If you are using [Middy](https://middy.js.org){target="_blank"} as your middlewa === "index.ts" - ```typescript hl_lines="18 29-33" + ```typescript hl_lines="22 36-40" --8<-- "docs/snippets/idempotency/makeHandlerIdempotent.ts" ``` @@ -392,7 +392,7 @@ If an error is thrown _outside_ the scope of the decorated function and after yo === "Handling exceptions" - ```typescript hl_lines="18-22 28 31" + ```typescript hl_lines="39-40 47-48" --8<-- "docs/snippets/idempotency/workingWithExceptions.ts" ``` @@ -606,7 +606,7 @@ This persistence layer is built-in, and you can either use an existing DynamoDB === "Customizing DynamoDBPersistenceLayer to suit your table structure" ```typescript hl_lines="7-15" - --8<-- "docs/snippets/idempotency/workingWithExceptions.ts" + --8<-- "docs/snippets/idempotency/customizePersistenceLayer.ts" ``` When using DynamoDB as a persistence layer, you can alter the attribute names by passing these parameters when initializing the persistence layer: @@ -658,7 +658,7 @@ You can enable in-memory caching with the **`useLocalCache`** parameter: === "Caching idempotent transactions in-memory to prevent multiple calls to storage" ```typescript hl_lines="12-13" - --8<-- "docs/snippets/idempotency/workingWithExceptions.ts" + --8<-- "docs/snippets/idempotency/workingWithLocalCache.ts" ``` When enabled, the default is to cache a maximum of 256 records in each Lambda execution environment - You can change it with the **`maxLocalCacheSize`** parameter. @@ -779,7 +779,7 @@ You can optionally set a static value for the partition key using the `staticPkV === "Reusing a DynamoDB table that uses a composite primary key" - ```typescript hl_lines="14-15" + ```typescript hl_lines="9" --8<-- "docs/snippets/idempotency/workingWithCompositeKey.ts" ``` From c4536a8f256c938273895c4730e819115279f069 Mon Sep 17 00:00:00 2001 From: Andrea Amorosi Date: Mon, 10 Jul 2023 18:17:23 +0200 Subject: [PATCH 7/7] chore: remove CDK mention --- docs/utilities/idempotency.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 6078cb7978..9341869b68 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -57,7 +57,7 @@ classDiagram Your Lambda function IAM Role must have `dynamodb:GetItem`, `dynamodb:PutItem`, `dynamodb:UpdateItem` and `dynamodb:DeleteItem` IAM permissions before using this feature. ???+ note - If you're using one of our examples: [AWS Serverless Application Model (SAM)](#required-resources), [AWS Cloud Development Kit (CDK)](#required-resources), or [Terraform](#required-resources) the required permissions are already included. + If you're using one of our examples: [AWS Serverless Application Model (SAM)](#required-resources) or [Terraform](#required-resources) the required permissions are already included. ### Required resources