feat: custom runtimes, optional VPC, python 3.11

lib/bootstrapper/index.ts

@@ -10,15 +10,14 @@ import {
   RemovalPolicy,
 } from "aws-cdk-lib";
 import { Construct } from "constructs";
+import { CustomLambdaFunctionOptions } from "../utils";
 
 function hasVpc(
   instance: aws_rds.DatabaseInstance | aws_rds.IDatabaseInstance
 ): instance is aws_rds.DatabaseInstance {
   return (instance as aws_rds.DatabaseInstance).vpc !== undefined;
 }
 
-const DEFAULT_PGSTAC_VERSION = "0.6.13";
-
 /**
  * Bootstraps a database instance, installing pgSTAC onto the database.
  */
@@ -28,17 +27,18 @@ export class BootstrapPgStac extends Construct {
   constructor(scope: Construct, id: string, props: BootstrapPgStacProps) {
     super(scope, id);
 
-    const { pgstacVersion = DEFAULT_PGSTAC_VERSION } = props;
     const handler = new aws_lambda.Function(this, "lambda", {
-      handler: "handler.handler",
-      runtime: aws_lambda.Runtime.PYTHON_3_8,
-      code: aws_lambda.Code.fromDockerBuild(__dirname, {
+      ...props.lambdaFunctionOptions ?? {
+        runtime: aws_lambda.Runtime.PYTHON_3_8,
+        handler: "handler.handler",
+        memorySize: 128,
+        logRetention: aws_logs.RetentionDays.ONE_WEEK,
+        timeout: Duration.minutes(2)
+      },
+      code: props.lambdaAssetCode ?? aws_lambda.Code.fromDockerBuild(__dirname, {
         file: "runtime/Dockerfile",
-        buildArgs: { PGSTAC_VERSION: pgstacVersion },
       }),
-      timeout: Duration.minutes(2),
       vpc: hasVpc(props.database) ? props.database.vpc : props.vpc,
-      logRetention: aws_logs.RetentionDays.ONE_WEEK,
     });
 
     this.secret = new aws_secretsmanager.Secret(this, "secret", {
@@ -75,9 +75,6 @@ export class BootstrapPgStac extends Construct {
     new CustomResource(this, "bootstrapper", {
       serviceToken: handler.functionArn,
       properties: {
-        // By setting pgstac_version in the properties assures
-        // that Create/Update events will be passed to the service token
-        pgstac_version: pgstacVersion,
         conn_secret_arn: props.dbSecret.secretArn,
         new_user_secret_arn: this.secret.secretArn,
       },
@@ -127,16 +124,22 @@ export interface BootstrapPgStacProps {
   readonly pgstacUsername?: string;
 
   /**
-   * pgSTAC version to be installed.
+   * Prefix to assign to the generated `secrets_manager.Secret`
    *
-   * @default 0.6.8
+   * @default pgstac
    */
-  readonly pgstacVersion?: string;
+  readonly secretsPrefix?: string;
 
   /**
-   * Prefix to assign to the generated `secrets_manager.Secret`
+   * Optional settings for the lambda function.
    *
-   * @default pgstac
+   * @default - defined in the construct.
    */
-  readonly secretsPrefix?: string;
+  readonly lambdaFunctionOptions?: CustomLambdaFunctionOptions;
+
+  /**
+   * Optional lambda asset code
+   * @default default runtime defined in this repository
+   */
+  readonly lambdaAssetCode?: aws_lambda.AssetCode;
 }

lib/bootstrapper/runtime/Dockerfile

@@ -1,11 +1,8 @@
 FROM lambci/lambda:build-python3.8
 
-ARG PGSTAC_VERSION
-RUN echo "Using PGSTAC Version ${PGSTAC_VERSION}"
-
 WORKDIR /tmp
 
-RUN pip install httpx psycopg[binary,pool] pypgstac==${PGSTAC_VERSION} -t /asset
+RUN pip install httpx psycopg[binary,pool] pypgstac==0.6.13 -t /asset
 
 COPY runtime/handler.py /asset/handler.py
 

lib/database/index.ts

@@ -50,7 +50,6 @@ export class PgStacDatabase extends Construct {
       database: this.db,
       dbSecret: this.db.secret!,
       pgstacDbName: props.pgstacDbName,
-      pgstacVersion: props.pgstacVersion,
       pgstacUsername: props.pgstacUsername,
       secretsPrefix: props.secretsPrefix,
     });
@@ -100,7 +99,6 @@ export class PgStacDatabase extends Construct {
 
 export interface PgStacDatabaseProps extends rds.DatabaseInstanceProps {
   readonly pgstacDbName?: BootstrapPgStacProps["pgstacDbName"];
-  readonly pgstacVersion?: BootstrapPgStacProps["pgstacVersion"];
   readonly pgstacUsername?: BootstrapPgStacProps["pgstacUsername"];
   readonly secretsPrefix?: BootstrapPgStacProps["secretsPrefix"];
 }

lib/index.ts

@@ -6,3 +6,4 @@ export * from "./stac-api";
 export * from "./titiler-pgstac-api";
 export * from "./stac-browser";
 export * from "./tipg-api";
+export * from "./utils";

lib/ingestor-api/index.ts

@@ -4,15 +4,16 @@ import {
   aws_ec2 as ec2,
   aws_iam as iam,
   aws_lambda as lambda,
+  aws_logs,
   aws_lambda_event_sources as events,
   aws_secretsmanager as secretsmanager,
   aws_ssm as ssm,
   Duration,
   RemovalPolicy,
   Stack,
 } from "aws-cdk-lib";
-import { PythonFunction, PythonFunctionProps } from "@aws-cdk/aws-lambda-python-alpha";
 import { Construct } from "constructs";
+import { CustomLambdaFunctionOptions } from "../utils";
 
 export class StacIngestor extends Construct {
   table: dynamodb.Table;
@@ -55,7 +56,8 @@ export class StacIngestor extends Construct {
       dbVpc: props.vpc,
       dbSecurityGroup: props.stacDbSecurityGroup,
       subnetSelection: props.subnetSelection,
-      apiCode: props.apiCode,
+      lambdaAssetCode: props.apiLambdaAssetCode,
+      lambdaFunctionOptions: props.apiLambdaFunctionOptions,
     });
 
     this.buildApiEndpoint({
@@ -73,7 +75,8 @@ export class StacIngestor extends Construct {
       dbVpc: props.vpc,
       dbSecurityGroup: props.stacDbSecurityGroup,
       subnetSelection: props.subnetSelection,
-      ingestorCode: props.ingestorCode,
+      lambdaAssetCode: props.ingestorLambdaAssetCode,
+      lambdaFunctionOptions: props.ingestorLambdaFunctionOptions
     });
 
     this.registerSsmParameter({
@@ -110,25 +113,27 @@ export class StacIngestor extends Construct {
     dbVpc: ec2.IVpc;
     dbSecurityGroup: ec2.ISecurityGroup;
     subnetSelection: ec2.SubnetSelection
-    apiCode?: ApiCode;
-  }): PythonFunction {
-
-    const apiCode = props.apiCode || {
-      entry: `${__dirname}/runtime`,
-      index: "src/handler.py",
-      handler: "handler",
-    };
-
-    const handler = new PythonFunction(this, "api-handler", {
-      ...apiCode,
-      runtime: lambda.Runtime.PYTHON_3_9,
-      timeout: Duration.seconds(30),
-      environment: { DB_SECRET_ARN: props.dbSecret.secretArn, ...props.env },
+    lambdaFunctionOptions?: CustomLambdaFunctionOptions;
+    lambdaAssetCode?: lambda.AssetCode;
+  }): lambda.Function {
+
+    const handler = new lambda.Function(this, "api-handler", {
+      ...props.lambdaFunctionOptions ?? {
+        runtime: lambda.Runtime.PYTHON_3_9,
+        handler: "handler.handler",
+        memorySize: 2048,
+        logRetention: aws_logs.RetentionDays.ONE_WEEK,
+        timeout: Duration.seconds(30)
+      },
+      code: props.lambdaAssetCode ?? lambda.Code.fromDockerBuild(__dirname, {
+        file: "runtime/Dockerfile",
+        buildArgs: { PYTHON_VERSION: '3.9' },
+      }),
       vpc: props.dbVpc,
       vpcSubnets: props.subnetSelection,
       allowPublicSubnet: true,
-      role: this.handlerRole,
-      memorySize: 2048,
+      environment: { DB_SECRET_ARN: props.dbSecret.secretArn, ...props.env },
+      role: this.handlerRole
     });
 
     // Allow handler to read DB secret
@@ -153,26 +158,28 @@ export class StacIngestor extends Construct {
     dbVpc: ec2.IVpc;
     dbSecurityGroup: ec2.ISecurityGroup;
     subnetSelection: ec2.SubnetSelection;
-    ingestorCode?: IngestorCode;
-  }): PythonFunction {
-
-
-
-    const ingestorCode = props.ingestorCode || {
-      entry: `${__dirname}/runtime`,
-      index: "src/ingestor.py",
-      handler: "handler",
-    };
+    lambdaFunctionOptions?: CustomLambdaFunctionOptions;
+    lambdaAssetCode?: lambda.AssetCode;
+  }): lambda.Function {
 
-    const handler = new PythonFunction(this, "stac-ingestor", {
-      ...ingestorCode,
-      runtime: lambda.Runtime.PYTHON_3_9,
-      timeout: Duration.seconds(180),
-      environment: { DB_SECRET_ARN: props.dbSecret.secretArn, ...props.env },
+
+    const handler = new lambda.Function(this, "stac-ingestor", {
+      ...props.lambdaFunctionOptions ?? {
+        runtime: lambda.Runtime.PYTHON_3_9,
+        handler: "handler.handler",
+        memorySize: 2048,
+        logRetention: aws_logs.RetentionDays.ONE_WEEK,
+        timeout: Duration.seconds(180)
+      },
+      code: props.lambdaAssetCode ?? lambda.Code.fromDockerBuild(__dirname, {
+        file: "runtime/Dockerfile",
+        buildArgs: { PYTHON_VERSION: '3.9' },
+      }),
       vpc: props.dbVpc,
       vpcSubnets: props.subnetSelection,
       allowPublicSubnet: true,
-      memorySize: 2048,
+      environment: { DB_SECRET_ARN: props.dbSecret.secretArn, ...props.env },
+      role: this.handlerRole
     });
 
     // Allow handler to read DB secret
@@ -309,54 +316,29 @@ export interface StacIngestorProps {
    readonly ingestorDomainNameOptions?: apigateway.DomainNameOptions;
 
   /**
-   * Custom code for the ingestor api.
-   *
-   * @default - default in the runtime folder. 
+   * Optional api lambda asset code
+   * @default - default runtime defined in this repository.
    */
-   readonly apiCode?: ApiCode;
-
-   /**
-   * Custom code for the ingestor.
-   *
-   * @default - default in the runtime folder. 
-   */
-   readonly ingestorCode?: IngestorCode;
-}
-
-export interface ApiCode {
+  readonly apiLambdaAssetCode?: lambda.AssetCode;
 
   /**
-   * Path to the source of the function or the location for dependencies, for the api lambda. 
+   * Optional ingestor lambda asset code
+   * @default - default runtime defined in this repository.
    */
-  readonly entry: PythonFunctionProps["entry"];
+  readonly ingestorLambdaAssetCode?: lambda.AssetCode;
 
   /**
-   * Path to the index file containing the exported handler, relative to `api_lambda_entry`. 
-   */
-  readonly index: PythonFunctionProps["index"];
-
-  /**
-   * The name of the exported handler in the `api_lambda_index` file. 
-   */
-  readonly handler: PythonFunctionProps["handler"];
-
-}
-
-export interface IngestorCode {
+     * Optional settings for the api lambda function.
+     *
+     * @default - defined in the construct.
+     */
+  readonly apiLambdaFunctionOptions?: CustomLambdaFunctionOptions;
 
   /**
-   * Path to the source of the function or the location for dependencies, for the ingestor lambda. 
-   */
-  readonly entry: PythonFunctionProps["entry"];
-
-  /**
-   * Path to the index file containing the exported handler, relative to `ingestor_lambda_entry`. 
-   */
-  readonly index: PythonFunctionProps["index"];
-
-  /**
-   * The name of the exported handler in the `ingestor_lambda_index` file.
-   */
-  readonly handler: PythonFunctionProps["handler"];
-
+     * Optional settings for the ingestor lambda function.
+     *
+     * @default - defined in the construct.
+     */
+readonly ingestorLambdaFunctionOptions?: CustomLambdaFunctionOptions;
+
 }

lib/ingestor-api/runtime/Dockerfile

@@ -0,0 +1,17 @@
+FROM public.ecr.aws/lambda/python:3.10
+
+WORKDIR /tmp
+RUN python -m pip install pip -U
+
+COPY runtime/requirements.txt requirements.txt
+RUN python -m pip install -r requirements.txt "mangum>=0.14,<0.15" -t /asset --no-binary pydantic
+
+# Reduce package size and remove useless files
+RUN cd /asset && find . -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-[0-9]*//'); cp $f $n; done;
+RUN cd /asset && find . -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf
+RUN cd /asset && find . -type f -a -name '*.py' -print0 | xargs -0 rm -f
+RUN find /asset -type d -a -name 'tests' -print0 | xargs -0 rm -rf
+
+COPY runtime/src/*.py /asset/
+
+CMD ["echo", "hello world"]

lib/ingestor-api/runtime/requirements.txt

@@ -1,7 +1,6 @@
 Authlib==1.0.1
 cachetools==5.1.0
 fastapi>=0.75.1
-mangum>=0.15.0
 orjson>=3.6.8
 psycopg[binary,pool]>=3.0.15
 pydantic_ssm_settings>=0.2.0