diff --git a/lib/ingestor-api/index.ts b/lib/ingestor-api/index.ts index c2de4d8..dc65f02 100644 --- a/lib/ingestor-api/index.ts +++ b/lib/ingestor-api/index.ts @@ -11,7 +11,7 @@ import { RemovalPolicy, Stack, } from "aws-cdk-lib"; -import { PythonFunction } from "@aws-cdk/aws-lambda-python-alpha"; +import { PythonFunction, PythonFunctionProps } from "@aws-cdk/aws-lambda-python-alpha"; import { Construct } from "constructs"; export class StacIngestor extends Construct { @@ -55,6 +55,7 @@ export class StacIngestor extends Construct { dbVpc: props.vpc, dbSecurityGroup: props.stacDbSecurityGroup, subnetSelection: props.subnetSelection, + apiCode: props.apiCode, }); this.buildApiEndpoint({ @@ -72,6 +73,7 @@ export class StacIngestor extends Construct { dbVpc: props.vpc, dbSecurityGroup: props.stacDbSecurityGroup, subnetSelection: props.subnetSelection, + ingestorCode: props.ingestorCode, }); this.registerSsmParameter({ @@ -108,11 +110,17 @@ export class StacIngestor extends Construct { dbVpc: ec2.IVpc; dbSecurityGroup: ec2.ISecurityGroup; subnetSelection: ec2.SubnetSelection + apiCode?: ApiCode; }): PythonFunction { - const handler = new PythonFunction(this, "api-handler", { + const apiCode = props.apiCode || { entry: `${__dirname}/runtime`, index: "src/handler.py", + handler: "handler", + }; + + const handler = new PythonFunction(this, "api-handler", { + ...apiCode, runtime: lambda.Runtime.PYTHON_3_9, timeout: Duration.seconds(30), environment: { DB_SECRET_ARN: props.dbSecret.secretArn, ...props.env }, @@ -145,10 +153,19 @@ export class StacIngestor extends Construct { dbVpc: ec2.IVpc; dbSecurityGroup: ec2.ISecurityGroup; subnetSelection: ec2.SubnetSelection; + ingestorCode?: IngestorCode; }): PythonFunction { - const handler = new PythonFunction(this, "stac-ingestor", { + + + + const ingestorCode = props.ingestorCode || { entry: `${__dirname}/runtime`, index: "src/ingestor.py", + handler: "handler", + }; + + const handler = new PythonFunction(this, "stac-ingestor", { + ...ingestorCode, runtime: lambda.Runtime.PYTHON_3_9, timeout: Duration.seconds(180), environment: { DB_SECRET_ARN: props.dbSecret.secretArn, ...props.env }, @@ -290,4 +307,56 @@ export interface StacIngestorProps { * Custom Domain Name Options for Ingestor API */ readonly ingestorDomainNameOptions?: apigateway.DomainNameOptions; + + /** + * Custom code for the ingestor api. + * + * @default - default in the runtime folder. + */ + readonly apiCode?: ApiCode; + + /** + * Custom code for the ingestor. + * + * @default - default in the runtime folder. + */ + readonly ingestorCode?: IngestorCode; } + +export interface ApiCode { + + /** + * Path to the source of the function or the location for dependencies, for the api lambda. + */ + readonly entry: PythonFunctionProps["entry"]; + + /** + * Path to the index file containing the exported handler, relative to `api_lambda_entry`. + */ + readonly index: PythonFunctionProps["index"]; + + /** + * The name of the exported handler in the `api_lambda_index` file. + */ + readonly handler: PythonFunctionProps["handler"]; + +} + +export interface IngestorCode { + + /** + * Path to the source of the function or the location for dependencies, for the ingestor lambda. + */ + readonly entry: PythonFunctionProps["entry"]; + + /** + * Path to the index file containing the exported handler, relative to `ingestor_lambda_entry`. + */ + readonly index: PythonFunctionProps["index"]; + + /** + * The name of the exported handler in the `ingestor_lambda_index` file. + */ + readonly handler: PythonFunctionProps["handler"]; + +} \ No newline at end of file diff --git a/lib/tipg-api/index.ts b/lib/tipg-api/index.ts index 23dcb39..c9f7fe7 100644 --- a/lib/tipg-api/index.ts +++ b/lib/tipg-api/index.ts @@ -87,9 +87,9 @@ import { readonly dbSecret: secretsmanager.ISecret; /** - * Custom code to run for fastapi-pgstac. + * Custom code to run for the application. * - * @default - simplified version of fastapi-pgstac + * @default - simplified version of tipg. */ readonly apiCode?: TiPgApiEntrypoint; diff --git a/lib/titiler-pgstac-api/index.ts b/lib/titiler-pgstac-api/index.ts index a64cbb3..0a09fb0 100644 --- a/lib/titiler-pgstac-api/index.ts +++ b/lib/titiler-pgstac-api/index.ts @@ -8,51 +8,61 @@ import { CfnOutput, Duration, aws_logs, + BundlingOptions } from "aws-cdk-lib"; + import { Runtime } from 'aws-cdk-lib/aws-lambda'; + import {PythonFunction} from "@aws-cdk/aws-lambda-python-alpha"; import { IDomainName, HttpApi } from "@aws-cdk/aws-apigatewayv2-alpha"; import { HttpLambdaIntegration } from "@aws-cdk/aws-apigatewayv2-integrations-alpha"; import { Construct } from "constructs"; + + // default settings that can be overridden by the user-provided environment. + let defaultTitilerPgstacEnv :{ [key: string]: any } = { + "CPL_VSIL_CURL_ALLOWED_EXTENSIONS": ".tif,.TIF,.tiff", + "GDAL_CACHEMAX": "200", + "GDAL_DISABLE_READDIR_ON_OPEN": "EMPTY_DIR", + "GDAL_INGESTED_BYTES_AT_OPEN": "32768", + "GDAL_HTTP_MERGE_CONSECUTIVE_RANGES": "YES", + "GDAL_HTTP_MULTIPLEX": "YES", + "GDAL_HTTP_VERSION": "2", + "PYTHONWARNINGS": "ignore", + "VSI_CACHE": "TRUE", + "VSI_CACHE_SIZE": "5000000", + "DB_MIN_CONN_SIZE": "1", + "DB_MAX_CONN_SIZE": "1" + } + export class TitilerPgstacApiLambda extends Construct { readonly url: string; public titilerPgstacLambdaFunction: lambda.Function; constructor(scope: Construct, id: string, props: TitilerPgStacApiLambdaProps) { super(scope, id); - - const titilerPgstacEnv = { - "CPL_VSIL_CURL_ALLOWED_EXTENSIONS": ".tif,.TIF,.tiff", - "GDAL_CACHEMAX": "200", - "GDAL_DISABLE_READDIR_ON_OPEN": "EMPTY_DIR", - "GDAL_INGESTED_BYTES_AT_OPEN": "32768", - "GDAL_HTTP_MERGE_CONSECUTIVE_RANGES": "YES", - "GDAL_HTTP_MULTIPLEX": "YES", - "GDAL_HTTP_VERSION": "2", - "PYTHONWARNINGS": "ignore", - "VSI_CACHE": "TRUE", - "VSI_CACHE_SIZE": "5000000", - "DB_MIN_CONN_SIZE": "1", - "DB_MAX_CONN_SIZE": "1", - "PGSTAC_SECRET_ARN": props.dbSecret.secretArn, - } - - - this.titilerPgstacLambdaFunction = new lambda.Function(this, "lambda", { - handler: "handler.handler", + + + // if user provided environment variables, merge them with the defaults. + const apiEnv = props.apiEnv ? { ...defaultTitilerPgstacEnv, ...props.apiEnv, "PGSTAC_SECRET_ARN": props.dbSecret.secretArn } : defaultTitilerPgstacEnv; + + const pythonLambdaOptions: TitilerPgstacPythonLambdaOptions = props.pythonLambdaOptions ?? { runtime: lambda.Runtime.PYTHON_3_10, - code: lambda.Code.fromDockerBuild(__dirname, { - file: "runtime/Dockerfile", - buildArgs: { PYTHON_VERSION: '3.10' }, - }), - timeout: Duration.seconds(30), + entry: `${__dirname}/runtime`, + index: "src/handler.py", + handler: "handler", + memorySize: 3008, + architecture: lambda.Architecture.X86_64 + } + + this.titilerPgstacLambdaFunction = new PythonFunction(this, "titiler-pgstac-api", { + ...pythonLambdaOptions, + environment: apiEnv, vpc: props.vpc, vpcSubnets: props.subnetSelection, allowPublicSubnet: true, - memorySize: 3008, logRetention: aws_logs.RetentionDays.ONE_WEEK, - environment: titilerPgstacEnv, - }); - + timeout: Duration.seconds(30) + }) + // grant access to buckets using addToRolePolicy if (props.buckets) { props.buckets.forEach(bucket => { @@ -105,8 +115,9 @@ import { readonly dbSecret: secretsmanager.ISecret; /** - * Customized environment variables to send to titiler-pgstac runtime. - */ + * Customized environment variables to send to titiler-pgstac runtime. These will be merged with `defaultTitilerPgstacEnv`. + * The database secret arn is automatically added to the environment variables at deployment. + /*/ readonly apiEnv?: Record; /** @@ -116,6 +127,57 @@ import { /** * Custom Domain Name Options for Titiler Pgstac API, + * + * @default - undefined. */ readonly titilerPgstacApiDomainName?: IDomainName; + + /** + * Optional settings for the titiler-pgstac python lambda function. + * + * @default - defined in the construct. + */ + readonly pythonLambdaOptions?: TitilerPgstacPythonLambdaOptions; + + } + + + export interface TitilerPgstacPythonLambdaOptions { + + /** + * Path to the source of the function or the location for dependencies. + */ + readonly entry: string; + /** + * The runtime environment. Only runtimes of the Python family are + * supported. + */ + readonly runtime: Runtime; + + /** + * The path (relative to entry) to the index file containing the exported handler. + * + */ + readonly index: string; + /** + * The name of the exported handler in the index file. + */ + readonly handler: string; + + /** + * Bundling options to use for this function. Use this to specify custom bundling options like + * the bundling Docker image, asset hash type, custom hash, architecture, etc. + */ + readonly bundling?: BundlingOptions; + + /** + * The amount of memory, in MB, that is allocated to your Lambda function. + */ + readonly memorySize: number; + + /** + * The system architectures compatible with this lambda function. + */ + readonly architecture: lambda.Architecture; + } diff --git a/lib/titiler-pgstac-api/runtime/Dockerfile b/lib/titiler-pgstac-api/runtime/Dockerfile deleted file mode 100644 index 54263ca..0000000 --- a/lib/titiler-pgstac-api/runtime/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -ARG PYTHON_VERSION - -FROM --platform=linux/amd64 public.ecr.aws/lambda/python:${PYTHON_VERSION} - -WORKDIR /tmp -RUN python -m pip install pip -U - -COPY runtime/requirements.txt requirements.txt -RUN python -m pip install -r requirements.txt "mangum>=0.14,<0.15" -t /asset - -# Reduce package size and remove useless files -RUN find /asset -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-[0-9]*//'); cp $f $n; done; -RUN find /asset -type d -name '__pycache__' -print0 | xargs -0 rm -rf -RUN find /asset -type f -name '*.py' -print0 | xargs -0 rm -f -RUN find /asset -type d -name 'tests' -print0 | xargs -0 rm -rf -RUN rm -rdf /asset/numpy/doc/ /asset/boto3* /asset/botocore* /asset/bin /asset/geos_license /asset/Misc - -COPY runtime/src/*.py /asset/ - -CMD ["echo", "hello world"] \ No newline at end of file diff --git a/lib/titiler-pgstac-api/runtime/dev_requirements.txt b/lib/titiler-pgstac-api/runtime/dev_requirements.txt index 400ebf5..5bb1c4c 100644 --- a/lib/titiler-pgstac-api/runtime/dev_requirements.txt +++ b/lib/titiler-pgstac-api/runtime/dev_requirements.txt @@ -1 +1,2 @@ -uvicorn \ No newline at end of file +uvicorn +boto3>=1.26.139 diff --git a/lib/titiler-pgstac-api/runtime/requirements.txt b/lib/titiler-pgstac-api/runtime/requirements.txt index 0659895..7a7838b 100644 --- a/lib/titiler-pgstac-api/runtime/requirements.txt +++ b/lib/titiler-pgstac-api/runtime/requirements.txt @@ -1,3 +1,3 @@ titiler.pgstac==0.5.1 -boto3>=1.26.139 psycopg[binary, pool] +mangum>=0.14,<0.15 diff --git a/lib/titiler-pgstac-api/runtime/src/handler.py b/lib/titiler-pgstac-api/runtime/src/handler.py index 2b5a987..23e8ed2 100644 --- a/lib/titiler-pgstac-api/runtime/src/handler.py +++ b/lib/titiler-pgstac-api/runtime/src/handler.py @@ -5,7 +5,7 @@ import asyncio import os from mangum import Mangum -from utils import get_secret_dict +from src.utils import get_secret_dict pgstac_secret_arn = os.environ["PGSTAC_SECRET_ARN"]