From e7eece1e96ed07ee7713263280c5ab451dbbdb61 Mon Sep 17 00:00:00 2001 From: Andrea Amorosi Date: Fri, 8 Mar 2024 17:51:23 +0100 Subject: [PATCH 1/4] docs(jmespath): add utility docs --- .../extractDataFromBuiltinEnvelope.json | 20 ++++++++ .../extractDataFromBuiltinEnvelope.ts | 21 ++++++++ .../jmespath/extractDataFromEnvelope.json | 8 +++ .../jmespath/extractDataFromEnvelope.ts | 31 +++++++++++ .../jmespath/powertoolsBase64GzipJmespath.ts | 13 +++++ .../powertoolsBase64GzipJmespathPayload.json | 3 ++ .../jmespath/powertoolsBase64Jmespath.ts | 13 +++++ .../powertoolsBase64JmespathPayload.json | 3 ++ .../jmespath/powertoolsCustomFunction.json | 9 ++++ .../jmespath/powertoolsCustomFunction.ts | 31 +++++++++++ .../powertoolsJsonIdempotencyJmespath.json | 30 +++++++++++ .../powertoolsJsonIdempotencyJmespath.ts | 51 +++++++++++++++++++ docs/snippets/tsconfig.json | 6 ++- mkdocs.yml | 2 + 14 files changed, 240 insertions(+), 1 deletion(-) create mode 100644 docs/snippets/jmespath/extractDataFromBuiltinEnvelope.json create mode 100644 docs/snippets/jmespath/extractDataFromBuiltinEnvelope.ts create mode 100644 docs/snippets/jmespath/extractDataFromEnvelope.json create mode 100644 docs/snippets/jmespath/extractDataFromEnvelope.ts create mode 100644 docs/snippets/jmespath/powertoolsBase64GzipJmespath.ts create mode 100644 docs/snippets/jmespath/powertoolsBase64GzipJmespathPayload.json create mode 100644 docs/snippets/jmespath/powertoolsBase64Jmespath.ts create mode 100644 docs/snippets/jmespath/powertoolsBase64JmespathPayload.json create mode 100644 docs/snippets/jmespath/powertoolsCustomFunction.json create mode 100644 docs/snippets/jmespath/powertoolsCustomFunction.ts create mode 100644 docs/snippets/jmespath/powertoolsJsonIdempotencyJmespath.json create mode 100644 docs/snippets/jmespath/powertoolsJsonIdempotencyJmespath.ts diff --git a/docs/snippets/jmespath/extractDataFromBuiltinEnvelope.json b/docs/snippets/jmespath/extractDataFromBuiltinEnvelope.json new file mode 100644 index 0000000000..9357e9d4b6 --- /dev/null +++ b/docs/snippets/jmespath/extractDataFromBuiltinEnvelope.json @@ -0,0 +1,20 @@ +{ + "Records": [ + { + "messageId": "19dd0b57-b21e-4ac1-bd88-01bbb068cb78", + "receiptHandle": "MessageReceiptHandle", + "body": "{\"customerId\":\"dd4649e6-2484-4993-acb8-0f9123103394\",\"booking\":{\"id\":\"5b2c4803-330b-42b7-811a-c68689425de1\",\"reference\":\"ySz7oA\",\"outboundFlightId\":\"20c0d2f2-56a3-4068-bf20-ff7703db552d\"},\"payment\":{\"receipt\":\"https://pay.stripe.com/receipts/acct_1Dvn7pF4aIiftV70/ch_3JTC14F4aIiftV700iFq2CHB/rcpt_K7QsrFln9FgFnzUuBIiNdkkRYGxUL0X\",\"amount\":100}}", + "attributes": { + "ApproximateReceiveCount": "1", + "SentTimestamp": "1523232000000", + "SenderId": "123456789012", + "ApproximateFirstReceiveTimestamp": "1523232000001" + }, + "messageAttributes": {}, + "md5OfBody": "7b270e59b47ff90a553787216d55d91d", + "eventSource": "aws:sqs", + "eventSourceARN": "arn:aws:sqs:us-east-1:123456789012:MyQueue", + "awsRegion": "us-east-1" + } + ] +} diff --git a/docs/snippets/jmespath/extractDataFromBuiltinEnvelope.ts b/docs/snippets/jmespath/extractDataFromBuiltinEnvelope.ts new file mode 100644 index 0000000000..6cd3102f7a --- /dev/null +++ b/docs/snippets/jmespath/extractDataFromBuiltinEnvelope.ts @@ -0,0 +1,21 @@ +import { + extractDataFromEnvelope, + SQS, +} from '@aws-lambda-powertools/jmespath/envelopes'; +import { Logger } from '@aws-lambda-powertools/logger'; +import type { SQSEvent } from 'aws-lambda'; + +const logger = new Logger(); + +type MessageBody = { + customerId: string; +}; + +export const handler = async (event: SQSEvent): Promise => { + const records = extractDataFromEnvelope>(event, SQS); + for (const record of records) { + // records is now a list containing the deserialized body of each message + const { customerId } = record; + logger.appendKeys({ customerId }); + } +}; diff --git a/docs/snippets/jmespath/extractDataFromEnvelope.json b/docs/snippets/jmespath/extractDataFromEnvelope.json new file mode 100644 index 0000000000..a802778bf7 --- /dev/null +++ b/docs/snippets/jmespath/extractDataFromEnvelope.json @@ -0,0 +1,8 @@ +{ + "body": "{\"customerId\":\"dd4649e6-2484-4993-acb8-0f9123103394\"}", + "deeplyNested": [ + { + "someData": [1, 2, 3] + } + ] +} diff --git a/docs/snippets/jmespath/extractDataFromEnvelope.ts b/docs/snippets/jmespath/extractDataFromEnvelope.ts new file mode 100644 index 0000000000..2d0f9bccf5 --- /dev/null +++ b/docs/snippets/jmespath/extractDataFromEnvelope.ts @@ -0,0 +1,31 @@ +import { extractDataFromEnvelope } from '@aws-lambda-powertools/jmespath/envelopes'; + +type MyEvent = { + body: string; // "{\"customerId\":\"dd4649e6-2484-4993-acb8-0f9123103394\"}" + deeplyNested: Array<{ someData: number[] }>; +}; + +type MessageBody = { + customerId: string; +}; + +export const handler = async (event: MyEvent): Promise => { + const payload = extractDataFromEnvelope( + event, + 'powertools_json(body)' + ); + const { customerId } = payload; // now deserialized + + // also works for fetching and flattening deeply nested data + const someData = extractDataFromEnvelope( + event, + 'deeplyNested[*].someData[]' + ); + + return { + customerId, + message: 'success', + context: someData, + statusCode: 200, + }; +}; diff --git a/docs/snippets/jmespath/powertoolsBase64GzipJmespath.ts b/docs/snippets/jmespath/powertoolsBase64GzipJmespath.ts new file mode 100644 index 0000000000..5400b9059a --- /dev/null +++ b/docs/snippets/jmespath/powertoolsBase64GzipJmespath.ts @@ -0,0 +1,13 @@ +import { extractDataFromEnvelope } from '@aws-lambda-powertools/jmespath/envelopes'; +import { Logger } from '@aws-lambda-powertools/logger'; + +const logger = new Logger(); + +export const handler = async (event: { payload: string }): Promise => { + const logGroup = extractDataFromEnvelope( + event, + 'powertools_base64_gzip(payload) | powertools_json(@).logGroup' + ); + + logger.info('Log group name', { logGroup }); +}; diff --git a/docs/snippets/jmespath/powertoolsBase64GzipJmespathPayload.json b/docs/snippets/jmespath/powertoolsBase64GzipJmespathPayload.json new file mode 100644 index 0000000000..470fb13c2e --- /dev/null +++ b/docs/snippets/jmespath/powertoolsBase64GzipJmespathPayload.json @@ -0,0 +1,3 @@ +{ + "payload": "H4sIACZAXl8C/52PzUrEMBhFX2UILpX8tPbHXWHqIOiq3Q1F0ubrWEiakqTWofTdTYYB0YWL2d5zvnuTFellBIOedoiyKH5M0iwnlKH7HZL6dDB6ngLDfLFYctUKjie9gHFaS/sAX1xNEq525QxwFXRGGMEkx4Th491rUZdV3YiIZ6Ljfd+lfSyAtZloacQgAkqSJCGhxM6t7cwwuUGPz4N0YKyvO6I9WDeMPMSo8Z4Ca/kJ6vMEYW5f1MX7W1lVxaG8vqX8hNFdjlc0iCBBSF4ERT/3Pl7RbMGMXF2KZMh/C+gDpNS7RRsp0OaRGzx0/t8e0jgmcczyLCWEePhni/23JWalzjdu0a3ZvgEaNLXeugEAAA==" +} diff --git a/docs/snippets/jmespath/powertoolsBase64Jmespath.ts b/docs/snippets/jmespath/powertoolsBase64Jmespath.ts new file mode 100644 index 0000000000..46e0d7bbf3 --- /dev/null +++ b/docs/snippets/jmespath/powertoolsBase64Jmespath.ts @@ -0,0 +1,13 @@ +import { extractDataFromEnvelope } from '@aws-lambda-powertools/jmespath/envelopes'; +import { Logger } from '@aws-lambda-powertools/logger'; + +const logger = new Logger(); + +export const handler = async (event: { payload: string }): Promise => { + const data = extractDataFromEnvelope( + event, + 'powertools_json(powertools_base64(payload))' + ); + + logger.info('Decoded payload', { data }); +}; diff --git a/docs/snippets/jmespath/powertoolsBase64JmespathPayload.json b/docs/snippets/jmespath/powertoolsBase64JmespathPayload.json new file mode 100644 index 0000000000..eae0118a5c --- /dev/null +++ b/docs/snippets/jmespath/powertoolsBase64JmespathPayload.json @@ -0,0 +1,3 @@ +{ + "payload": "eyJ1c2VyX2lkIjogMTIzLCAicHJvZHVjdF9pZCI6IDEsICJxdWFudGl0eSI6IDIsICJwcmljZSI6IDEwLjQwLCAiY3VycmVuY3kiOiAiVVNEIn0=" +} diff --git a/docs/snippets/jmespath/powertoolsCustomFunction.json b/docs/snippets/jmespath/powertoolsCustomFunction.json new file mode 100644 index 0000000000..0d098b0c78 --- /dev/null +++ b/docs/snippets/jmespath/powertoolsCustomFunction.json @@ -0,0 +1,9 @@ +{ + "Records": [ + { + "application": "app", + "datetime": "2022-01-01T00:00:00.000Z", + "notification": "GyYA+AXhZKk/K5DkanoQSTYpSKMwwxXh8DRWVo9A1hLqAQ==" + } + ] +} diff --git a/docs/snippets/jmespath/powertoolsCustomFunction.ts b/docs/snippets/jmespath/powertoolsCustomFunction.ts new file mode 100644 index 0000000000..6328cb3ca1 --- /dev/null +++ b/docs/snippets/jmespath/powertoolsCustomFunction.ts @@ -0,0 +1,31 @@ +import { fromBase64 } from '@aws-lambda-powertools/commons/utils/base64'; +import { extractDataFromEnvelope } from '@aws-lambda-powertools/jmespath/envelopes'; +import { PowertoolsFunctions } from '@aws-lambda-powertools/jmespath/functions'; +import { Logger } from '@aws-lambda-powertools/logger'; +import { brotliDecompressSync } from 'node:zlib'; + +const logger = new Logger(); + +// prettier-ignore +class CustomFunctions extends PowertoolsFunctions { + @PowertoolsFunctions.signature({ // (1)! + argumentsSpecs: [['string']], + variadic: false, + }) + public funcDecodeBrotliCompression(value: string): string { // (2)! + const encoded = fromBase64(value, 'base64'); + const uncompressed = brotliDecompressSync(encoded); + + return uncompressed.toString(); + } +} + +export const handler = async (event: { payload: string }): Promise => { + const message = extractDataFromEnvelope( + event, + 'Records[*].decode_brotli_compression(notification) | [*].powertools_json(@).message', + { customFunctions: new CustomFunctions() } + ); + + logger.info('Decoded message', { message }); +}; diff --git a/docs/snippets/jmespath/powertoolsJsonIdempotencyJmespath.json b/docs/snippets/jmespath/powertoolsJsonIdempotencyJmespath.json new file mode 100644 index 0000000000..0534d6bacd --- /dev/null +++ b/docs/snippets/jmespath/powertoolsJsonIdempotencyJmespath.json @@ -0,0 +1,30 @@ +{ + "version": "2.0", + "routeKey": "ANY /createpayment", + "rawPath": "/createpayment", + "rawQueryString": "", + "headers": { + "Header1": "value1", + "Header2": "value2" + }, + "requestContext": { + "accountId": "123456789012", + "apiId": "api-id", + "domainName": "id.execute-api.us-east-1.amazonaws.com", + "domainPrefix": "id", + "http": { + "method": "POST", + "path": "/createpayment", + "protocol": "HTTP/1.1", + "sourceIp": "ip", + "userAgent": "agent" + }, + "requestId": "id", + "routeKey": "ANY /createpayment", + "stage": "$default", + "time": "10/Feb/2021:13:40:43 +0000", + "timeEpoch": 1612964443723 + }, + "body": "{\"user\":\"xyz\",\"product_id\":\"123456789\"}", + "isBase64Encoded": false +} diff --git a/docs/snippets/jmespath/powertoolsJsonIdempotencyJmespath.ts b/docs/snippets/jmespath/powertoolsJsonIdempotencyJmespath.ts new file mode 100644 index 0000000000..5ce144a109 --- /dev/null +++ b/docs/snippets/jmespath/powertoolsJsonIdempotencyJmespath.ts @@ -0,0 +1,51 @@ +import { + IdempotencyConfig, + makeIdempotent, +} from '@aws-lambda-powertools/idempotency'; +import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; +import type { APIGatewayEvent } from 'aws-lambda'; +import { randomUUID } from 'node:crypto'; + +const persistenceStore = new DynamoDBPersistenceLayer({ + tableName: 'IdempotencyTable', +}); + +export const handler = makeIdempotent( + async (event: APIGatewayEvent) => { + const body = JSON.parse(event.body || '{}'); + const { user, productId } = body; + + const result = await createSubscriptionPayment(user, productId); + + return { + statusCode: 200, + body: JSON.stringify({ + paymentId: result.id, + message: 'success', + }), + }; + }, + { + persistenceStore, + config: new IdempotencyConfig({ + eventKeyJmesPath: 'powertools_json(body)', + }), + } +); + +const createSubscriptionPayment = async ( + user: string, + productId: string +): Promise<{ id: string; message: string }> => { + const payload = { user, productId }; + const response = await fetch('https://httpbin.org/anything', { + method: 'POST', + body: JSON.stringify(payload), + }); + + if (!response.ok) { + throw new Error('Failed to create subscription payment'); + } + + return { id: randomUUID(), message: 'paid' }; +}; diff --git a/docs/snippets/tsconfig.json b/docs/snippets/tsconfig.json index 1a3fe8b171..d6aec30ce7 100644 --- a/docs/snippets/tsconfig.json +++ b/docs/snippets/tsconfig.json @@ -27,7 +27,11 @@ "@aws-lambda-powertools/idempotency/middleware": [ "../../packages/idempotency/lib/middleware" ], - "@aws-lambda-powertools/batch": ["../../packages/batch/lib"] + "@aws-lambda-powertools/batch": ["../../packages/batch/lib"], + "@aws-lambda-powertools/jmespath": ["../../packages/jmespath/lib"], + "@aws-lambda-powertools/jmespath/envelopes": [ + "../../packages/jmespath/lib/envelopes" + ] } } } diff --git a/mkdocs.yml b/mkdocs.yml index e7556ff610..107ca2b45f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -19,6 +19,7 @@ nav: - utilities/parameters.md - utilities/idempotency.md - utilities/batch.md + - utilities/jmespath.md - Processes: - Roadmap: roadmap.md - Versioning policy: versioning.md @@ -100,6 +101,7 @@ plugins: glob: - snippets/node_modules/* - snippets/package.json + - snippets/CHANGELOG.md extra_css: - stylesheets/extra.css From 6b1382d211d8e49a82f592f58b80408623f66e4d Mon Sep 17 00:00:00 2001 From: Andrea Amorosi Date: Fri, 8 Mar 2024 18:22:41 +0100 Subject: [PATCH 2/4] docs: utility page --- docs/utilities/jmespath.md | 171 +++++++++++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 docs/utilities/jmespath.md diff --git a/docs/utilities/jmespath.md b/docs/utilities/jmespath.md new file mode 100644 index 0000000000..bb81c94fed --- /dev/null +++ b/docs/utilities/jmespath.md @@ -0,0 +1,171 @@ +--- +title: JMESPath Functions +description: Utility +--- + +???+ warning + This is an unreleased feature that is currently under active development and will be released soon. Please check back later for updates. + +???+ tip + JMESPath is a query language for JSON used by tools like the AWS CLI and Powertools for AWS Lambda (TypeScript). + +Built-in [JMESPath](https://jmespath.org/){target="_blank" rel="nofollow"} Functions to easily deserialize common encoded JSON payloads in Lambda functions. + +## Key features + +* Deserialize JSON from JSON strings, base64, and compressed data +* Use JMESPath to extract and combine data recursively +* Provides commonly used JMESPath expression with popular event sources + +## Getting started + +You might have events that contains encoded JSON payloads as string, base64, or even in compressed format. It is a common use case to decode and extract them partially or fully as part of your Lambda function invocation. + +Powertools for AWS Lambda (TypeScript) also have utilities like [idempotency](idempotency.md){target="_blank"} where you might need to extract a portion of your data before using them. + +???+ info "Terminology" + **Envelope** is the terminology we use for the **JMESPath expression** to extract your JSON object from your data input. We might use those two terms interchangeably. + +### Extracting data + +You can use the `extractDataFromEnvelope` function with any [JMESPath expression](https://jmespath.org/tutorial.html){target="_blank" rel="nofollow"}. + +???+ tip + Another common use case is to fetch deeply nested data, filter, flatten, and more. + +=== "extractDataFromBuiltinEnvelope.ts" + ```typescript hl_lines="1 13 17 20 22" + --8<-- "docs/snippets/jmespath/extractDataFromEnvelope.ts" + ``` + +=== "extractDataFromEnvelope.json" + + ```json + --8<-- "docs/snippets/jmespath/extractDataFromEnvelope.json" + ``` + +### Built-in envelopes + +We provide built-in envelopes for popular AWS Lambda event sources to easily decode and/or deserialize JSON objects. + +=== "extractDataFromBuiltinEnvelope.ts" + ```typescript hl_lines="2-3 15" + --8<-- "docs/snippets/jmespath/extractDataFromBuiltinEnvelope.ts" + ``` + +=== "extractDataFromBuiltinEnvelope.json" + + ```json hl_lines="6 15" + --8<-- "docs/snippets/jmespath/extractDataFromBuiltinEnvelope.json" + ``` + +These are all built-in envelopes you can use along with their expression as a reference: + +| Envelope | JMESPath expression | +| --------------------------------- | ----------------------------------------------------------------------------------------- | +| **`API_GATEWAY_HTTP`** | `powertools_json(body)` | +| **`API_GATEWAY_REST`** | `powertools_json(body)` | +| **`CLOUDWATCH_EVENTS_SCHEDULED`** | `detail` | +| **`CLOUDWATCH_LOGS`** | `awslogs.powertools_base64_gzip(data) | powertools_json(@).logEvents[*]` | +| **`EVENTBRIDGE`** | `detail` | +| **`KINESIS_DATA_STREAM`** | `Records[*].kinesis.powertools_json(powertools_base64(data))` | +| **`S3_EVENTBRIDGE_SQS`** | `Records[*].powertools_json(body).detail` | +| **`S3_KINESIS_FIREHOSE`** | `records[*].powertools_json(powertools_base64(data)).Records[0]` | +| **`S3_SNS_KINESIS_FIREHOSE`** | `records[*].powertools_json(powertools_base64(data)).powertools_json(Message).Records[0]` | +| **`S3_SNS_SQS`** | `Records[*].powertools_json(body).powertools_json(Message).Records[0]` | +| **`S3_SQS`** | `Records[*].powertools_json(body).Records[0]` | +| **`SNS`** | `Records[0].Sns.Message | powertools_json(@)` | +| **`SQS`** | `Records[*].powertools_json(body)` | + +???+ tip "Using SNS?" + If you don't require SNS metadata, enable [raw message delivery](https://docs.aws.amazon.com/sns/latest/dg/sns-large-payload-raw-message-delivery.html). It will reduce multiple payload layers and size, when using SNS in combination with other services (_e.g., SQS, S3, etc_). + +## Advanced + +### Built-in JMESPath functions + +You can use our built-in JMESPath functions within your envelope expression. They handle deserialization for common data formats found in AWS Lambda event sources such as JSON strings, base64, and uncompress gzip data. + +#### powertools_json function + +Use `powertools_json` function to decode any JSON string anywhere a JMESPath expression is allowed. + +> **Idempotency scenario** + +This sample will deserialize the JSON string within the `body` key before [Idempotency](./idempotency.md){target="_blank"} processes it. + +=== "powertoolsJsonIdempotencyJmespath.ts" + + ```ts hl_lines="31" + --8<-- "docs/snippets/jmespath/powertoolsJsonIdempotencyJmespath.ts" + ``` + +=== "powertoolsJsonIdempotencyJmespath.json" + + ```json hl_lines="28" + --8<-- "docs/snippets/jmespath/powertoolsJsonIdempotencyJmespath.json" + ``` + +#### powertools_base64 function + +Use `powertools_base64` function to decode any base64 data. + +This sample will decode the base64 value within the `data` key, and deserialize the JSON string before processing. + +=== "powertoolsBase64Jmespath.ts" + + ```ts hl_lines="9" + --8<-- "docs/snippets/jmespath/powertoolsBase64Jmespath.ts" + ``` + +=== "powertoolsBase64JmespathPayload.json" + + ```json + --8<-- "docs/snippets/jmespath/powertoolsBase64JmespathPayload.json" + ``` + +#### powertools_base64_gzip function + +Use `powertools_base64_gzip` function to decompress and decode base64 data. + +This sample will decompress and decode base64 data from Cloudwatch Logs, then use JMESPath pipeline expression to pass the result for decoding its JSON string. + +=== "powertoolsBase64GzipJmespath.ts" + + ```ts hl_lines="9" + --8<-- "docs/snippets/jmespath/powertoolsBase64GzipJmespath.ts" + ``` + +=== "powertoolsBase64GzipJmespathPayload.json" + + ```json + --8<-- "docs/snippets/jmespath/powertoolsBase64GzipJmespathPayload.json" + ``` + +### Bring your own JMESPath function + +???+ warning + This should only be used for advanced use cases where you have special formats not covered by the built-in functions. + +For special binary formats that you want to decode before processing, you can bring your own JMESPath function by extending the `PowertoolsFunctions` class. + +Here is an example of how to decompress messages compressed using the [Brotli compression algorithm](https://nodejs.org/api/zlib.html#zlibbrotlidecompressbuffer-options-callback){target="_blank" rel="nofollow"}: + +=== "PowertoolsCustomFunction.ts" + + ```ts hl_lines="3 9 25-26" + --8<-- + docs/snippets/jmespath/powertoolsCustomFunction.ts::8 + docs/snippets/jmespath/powertoolsCustomFunction.ts:10: + + --8<-- + ``` + + 1. The function signature can be enforced at runtime by using the `@Functions.signature` decorator. + 2. The name of the function must start with the `func` prefix. + +=== "powertoolsCustomFunction.json" + + ```json + --8<-- "docs/snippets/jmespath/powertoolsCustomFunction.json" + ``` \ No newline at end of file From 1f877ce4853ad56e65116415e8441549b4d6faef Mon Sep 17 00:00:00 2001 From: Andrea Amorosi Date: Thu, 21 Mar 2024 18:52:29 +0100 Subject: [PATCH 3/4] chore: add readme --- packages/jmespath/README.md | 218 +++++++++++++++++++++++++++++++++ packages/jmespath/typedoc.json | 1 + 2 files changed, 219 insertions(+) create mode 100644 packages/jmespath/README.md diff --git a/packages/jmespath/README.md b/packages/jmespath/README.md new file mode 100644 index 0000000000..f52ca3d47d --- /dev/null +++ b/packages/jmespath/README.md @@ -0,0 +1,218 @@ +# Powertools for AWS Lambda (TypeScript) - JMESPath Utility + +Powertools for AWS Lambda (TypeScript) is a developer toolkit to implement Serverless [best practices and increase developer velocity](https://docs.powertools.aws.dev/lambda/typescript/latest/#features). + +You can use the package in both TypeScript and JavaScript code bases. + +- [Intro](#intro) +- [Usage](#usage) + - [Basic usage](#basic-usage) + - [Extract data from envelopes](#extract-data-from-envelopes) + - [JMESPath custom functions](#jmespath-custom-functions) +- [Contribute](#contribute) +- [Roadmap](#roadmap) +- [Connect](#connect) +- [How to support Powertools for AWS Lambda (TypeScript)?](#how-to-support-powertools-for-aws-lambda-typescript) + - [Becoming a reference customer](#becoming-a-reference-customer) + - [Sharing your work](#sharing-your-work) + - [Using Lambda Layer](#using-lambda-layer) +- [License](#license) + +## Intro + +The JMESPath utility is a high-level function to parse and extract data from JSON objects using JMESPath expressions. + +## Usage + +To get started, install the library by running: + +```sh +npm i @aws-lambda-powertools/jmespath +``` + +### Basic usage + +At its core, the library provides a utility function to extract data from a JSON object using a JMESPath expression. + +```ts +import { search } from '@aws-lambda-powertools/jmespath'; +import { Logger } from '@aws-lambda-powertools/logger'; + +const logger = new Logger(); + +type MyEvent = { + foo: { + bar: string; + }; +} + +export const handler = async (event: MyEvent): Promise => { + const result = search(event, 'foo.bar'); + logger.info(result); // "baz" +}; +``` + +### Extract data from envelopes + +In some cases, you may want to extract data from an envelope. The library provides a utility function to help you work with envelopes and extract data from them. + +```ts +import { extractDataFromEnvelope } from '@aws-lambda-powertools/jmespath/envelopes'; + +type MyEvent = { + body: string; // "{\"customerId\":\"dd4649e6-2484-4993-acb8-0f9123103394\"}" + deeplyNested: Array<{ someData: number[] }>; +}; + +type MessageBody = { + customerId: string; +}; + +export const handler = async (event: MyEvent): Promise => { + const payload = extractDataFromEnvelope( + event, + 'powertools_json(body)' + ); + const { customerId } = payload; // now deserialized + + // also works for fetching and flattening deeply nested data + const someData = extractDataFromEnvelope( + event, + 'deeplyNested[*].someData[]' + ); + + return { + customerId, + message: 'success', + context: someData, + statusCode: 200, + }; +}; +``` + +The library provides [a set of built-in envelopes](https://docs.powertools.aws.dev/lambda/typescript/latest/utilities/jmespath/#built-in-envelopes) to help you extract data from common event sources, such as S3, SQS, and SNS, and more. + +```ts +import { + extractDataFromEnvelope, + SQS, +} from '@aws-lambda-powertools/jmespath/envelopes'; +import { Logger } from '@aws-lambda-powertools/logger'; +import type { SQSEvent } from 'aws-lambda'; + +const logger = new Logger(); + +type MessageBody = { + customerId: string; +}; + +export const handler = async (event: SQSEvent): Promise => { + const records = extractDataFromEnvelope>(event, SQS); + for (const record of records) { + // records is now a list containing the deserialized body of each message + const { customerId } = record; + logger.appendKeys({ customerId }); + } +}; +``` + +### JMESPath custom functions + +In addition to all the [built-in JMESPath functions](https://jmespath.org/specification.html#built-in-functions), the library provides custom functions to help you work with complex data structures. For example, you can use the `powertools_json` function to parse a JSON string, or the `powertools_base64` function to decode a base64-encoded string: + +```ts +import { extractDataFromEnvelope } from '@aws-lambda-powertools/jmespath/envelopes'; +import { Logger } from '@aws-lambda-powertools/logger'; + +const logger = new Logger(); + +export const handler = async (event: { payload: string }): Promise => { + const data = extractDataFromEnvelope( + event, + 'powertools_json(powertools_base64(payload))' + ); + + logger.info('Decoded payload', { data }); +}; +``` + +Finally, you can also extend the library with your own custom functions. Below an example of how to create a custom function to decode a Brotli-compressed string. + +```ts +import { fromBase64 } from '@aws-lambda-powertools/commons/utils/base64'; +import { extractDataFromEnvelope } from '@aws-lambda-powertools/jmespath/envelopes'; +import { PowertoolsFunctions } from '@aws-lambda-powertools/jmespath/functions'; +import { Logger } from '@aws-lambda-powertools/logger'; +import { brotliDecompressSync } from 'node:zlib'; + +const logger = new Logger(); + +class CustomFunctions extends PowertoolsFunctions { + @PowertoolsFunctions.signature({ + argumentsSpecs: [['string']], + variadic: false, + }) + public funcDecodeBrotliCompression(value: string): string { + const encoded = fromBase64(value, 'base64'); + const uncompressed = brotliDecompressSync(encoded); + + return uncompressed.toString(); + } +} + +export const handler = async (event: { payload: string }): Promise => { + const message = extractDataFromEnvelope( + event, + 'Records[*].decode_brotli_compression(notification) | [*].powertools_json(@).message', + { customFunctions: new CustomFunctions() } + ); + + logger.info('Decoded message', { message }); +}; +``` + +## Contribute + +If you are interested in contributing to this project, please refer to our [Contributing Guidelines](https://github.com/aws-powertools/powertools-lambda-typescript/blob/main/CONTRIBUTING.md). + +## Roadmap + +The roadmap of Powertools for AWS Lambda (TypeScript) is driven by customers’ demand. +Help us prioritize upcoming functionalities or utilities by [upvoting existing RFCs and feature requests](https://github.com/aws-powertools/powertools-lambda-typescript/issues), or [creating new ones](https://github.com/aws-powertools/powertools-lambda-typescript/issues/new/choose), in this GitHub repository. + +## Connect + +* **Powertools for AWS Lambda on Discord**: `#typescript` - **[Invite link](https://discord.gg/B8zZKbbyET)** +* **Email**: aws-lambda-powertools-feedback@amazon.com + +## How to support Powertools for AWS Lambda (TypeScript)? + +### Becoming a reference customer + +Knowing which companies are using this library is important to help prioritize the project internally. If your company is using Powertools for AWS Lambda (TypeScript), you can request to have your name and logo added to the README file by raising a [Support Powertools for AWS Lambda (TypeScript) (become a reference)](https://github.com/aws-powertools/powertools-lambda-typescript/issues/new?assignees=&labels=customer-reference&template=support_powertools.yml&title=%5BSupport+Lambda+Powertools%5D%3A+%3Cyour+organization+name%3E) issue. + +The following companies, among others, use Powertools: + +* [Hashnode](https://hashnode.com/) +* [Trek10](https://www.trek10.com/) +* [Elva](https://elva-group.com) +* [globaldatanet](https://globaldatanet.com/) +* [Bailey Nelson](https://www.baileynelson.com.au) +* [Perfect Post](https://www.perfectpost.fr) +* [Sennder](https://sennder.com/) +* [Certible](https://www.certible.com/) +* [tecRacer GmbH & Co. KG](https://www.tecracer.com/) +* [AppYourself](https://appyourself.net) +* [Alma Media](https://www.almamedia.fi) + +### Sharing your work + +Share what you did with Powertools for AWS Lambda (TypeScript) 💞💞. Blog post, workshops, presentation, sample apps and others. Check out what the community has already shared about Powertools for AWS Lambda (TypeScript) [here](https://docs.powertools.aws.dev/lambda/typescript/latest/we_made_this). + +### Using Lambda Layer + +This helps us understand who uses Powertools for AWS Lambda (TypeScript) in a non-intrusive way, and helps us gain future investments for other Powertools for AWS Lambda languages. When [using Layers](https://docs.powertools.aws.dev/lambda/typescript/latest/#lambda-layer), you can add Powertools as a dev dependency to not impact the development process. + +## License + +This library is licensed under the MIT-0 License. See the LICENSE file. \ No newline at end of file diff --git a/packages/jmespath/typedoc.json b/packages/jmespath/typedoc.json index 737729805a..da81672090 100644 --- a/packages/jmespath/typedoc.json +++ b/packages/jmespath/typedoc.json @@ -6,6 +6,7 @@ "./src/index.ts", "./src/types.ts", "./src/envelopes.ts", + "./src/Functions.ts", "./src/PowertoolsFunctions.ts", ], "readme": "README.md" From a2173cd464cc41f7af7badc07f9f16e84b048a81 Mon Sep 17 00:00:00 2001 From: Andrea Amorosi Date: Sun, 24 Mar 2024 20:33:58 +0100 Subject: [PATCH 4/4] chore: address review feedback --- .../jmespath/powertoolsBase64GzipJmespath.ts | 4 +- .../jmespath/powertoolsBase64Jmespath.ts | 2 +- docs/utilities/jmespath.md | 56 ++++++++++++++++--- 3 files changed, 50 insertions(+), 12 deletions(-) diff --git a/docs/snippets/jmespath/powertoolsBase64GzipJmespath.ts b/docs/snippets/jmespath/powertoolsBase64GzipJmespath.ts index 5400b9059a..8fb360124f 100644 --- a/docs/snippets/jmespath/powertoolsBase64GzipJmespath.ts +++ b/docs/snippets/jmespath/powertoolsBase64GzipJmespath.ts @@ -5,9 +5,9 @@ const logger = new Logger(); export const handler = async (event: { payload: string }): Promise => { const logGroup = extractDataFromEnvelope( - event, + event, // (1)! 'powertools_base64_gzip(payload) | powertools_json(@).logGroup' ); - logger.info('Log group name', { logGroup }); + logger.info('Log group name', { logGroup }); // (2)! }; diff --git a/docs/snippets/jmespath/powertoolsBase64Jmespath.ts b/docs/snippets/jmespath/powertoolsBase64Jmespath.ts index 46e0d7bbf3..f9abd6ad33 100644 --- a/docs/snippets/jmespath/powertoolsBase64Jmespath.ts +++ b/docs/snippets/jmespath/powertoolsBase64Jmespath.ts @@ -9,5 +9,5 @@ export const handler = async (event: { payload: string }): Promise => { 'powertools_json(powertools_base64(payload))' ); - logger.info('Decoded payload', { data }); + logger.info('Decoded payload', { data }); // (1)! }; diff --git a/docs/utilities/jmespath.md b/docs/utilities/jmespath.md index bb81c94fed..b8d24e7f66 100644 --- a/docs/utilities/jmespath.md +++ b/docs/utilities/jmespath.md @@ -6,20 +6,17 @@ description: Utility ???+ warning This is an unreleased feature that is currently under active development and will be released soon. Please check back later for updates. -???+ tip - JMESPath is a query language for JSON used by tools like the AWS CLI and Powertools for AWS Lambda (TypeScript). - -Built-in [JMESPath](https://jmespath.org/){target="_blank" rel="nofollow"} Functions to easily deserialize common encoded JSON payloads in Lambda functions. +Built-in [JMESPath](https://jmespath.org/){target="_blank" rel="nofollow"} functions to easily deserialize common encoded JSON payloads in Lambda functions. ## Key features * Deserialize JSON from JSON strings, base64, and compressed data * Use JMESPath to extract and combine data recursively -* Provides commonly used JMESPath expression with popular event sources +* Provides commonly used JMESPath expressions with popular event sources ## Getting started -You might have events that contains encoded JSON payloads as string, base64, or even in compressed format. It is a common use case to decode and extract them partially or fully as part of your Lambda function invocation. +You might have events that contain encoded JSON payloads as string, base64, or even in compressed format. It is a common use case to decode and extract them partially or fully as part of your Lambda function invocation. Powertools for AWS Lambda (TypeScript) also have utilities like [idempotency](idempotency.md){target="_blank"} where you might need to extract a portion of your data before using them. @@ -86,7 +83,7 @@ These are all built-in envelopes you can use along with their expression as a re You can use our built-in JMESPath functions within your envelope expression. They handle deserialization for common data formats found in AWS Lambda event sources such as JSON strings, base64, and uncompress gzip data. -#### powertools_json function +#### `powertools_json` function Use `powertools_json` function to decode any JSON string anywhere a JMESPath expression is allowed. @@ -106,7 +103,7 @@ This sample will deserialize the JSON string within the `body` key before [Idemp --8<-- "docs/snippets/jmespath/powertoolsJsonIdempotencyJmespath.json" ``` -#### powertools_base64 function +#### `powertools_base64` function Use `powertools_base64` function to decode any base64 data. @@ -118,13 +115,24 @@ This sample will decode the base64 value within the `data` key, and deserialize --8<-- "docs/snippets/jmespath/powertoolsBase64Jmespath.ts" ``` + 1. The `data` variable contains the decoded object that looks like this: + ```json + { + user_id: 123, + product_id: 1, + quantity: 2, + price: 10.4, + currency: 'USD', + } + ``` + === "powertoolsBase64JmespathPayload.json" ```json --8<-- "docs/snippets/jmespath/powertoolsBase64JmespathPayload.json" ``` -#### powertools_base64_gzip function +#### `powertools_base64_gzip` function Use `powertools_base64_gzip` function to decompress and decode base64 data. @@ -136,6 +144,36 @@ This sample will decompress and decode base64 data from Cloudwatch Logs, then us --8<-- "docs/snippets/jmespath/powertoolsBase64GzipJmespath.ts" ``` + 1. The `payload` key contains a JSON object that once decompressed and decoded looks like this: + ```json + { + "owner": "123456789012", + "logGroup": "/aws/lambda/powertools-example", + "logStream": "2020/09/02/[$LATEST]d3a8dcaffc7f4de2b8db132e3e106660", + "subscriptionFilters": ["Destination"], + "messageType": "DATA_MESSAGE", + "logEvents": [ + { + "id": "eventId1", + "message": { + "username": "lessa", + "message": "hello world" + }, + "timestamp": 1440442987000 + }, + { + "id": "eventId2", + "message": { + "username": "dummy", + "message": "hello world" + }, + "timestamp": 1440442987001 + } + ] + } + ``` + 2. The `logGroup` variable contains the string `"/aws/lambda/powertools-example"`. + === "powertoolsBase64GzipJmespathPayload.json" ```json