From d1f5cfeeda284f5957098a124d1dbbc04d1f8c33 Mon Sep 17 00:00:00 2001 From: emileten Date: Thu, 12 Oct 2023 17:42:48 +0900 Subject: [PATCH] import eoapi-cdk constructs and reuse them --- infrastructure/aws/cdk/app.py | 370 ++++++++---------------- infrastructure/aws/cdk/config.py | 5 +- infrastructure/aws/requirements-cdk.txt | 8 +- 3 files changed, 133 insertions(+), 250 deletions(-) diff --git a/infrastructure/aws/cdk/app.py b/infrastructure/aws/cdk/app.py index b327440..45a435f 100644 --- a/infrastructure/aws/cdk/app.py +++ b/infrastructure/aws/cdk/app.py @@ -1,19 +1,14 @@ """ CDK Stack definition code for EOAPI """ -import json import os from typing import Any -from aws_cdk import App, CfnOutput, CustomResource, Duration, RemovalPolicy, Stack, Tags -from aws_cdk import aws_apigatewayv2_alpha as apigw +from aws_cdk import App, CfnOutput, Duration, RemovalPolicy, Stack, Tags from aws_cdk import aws_ec2 as ec2 -from aws_cdk import aws_iam as iam from aws_cdk import aws_lambda from aws_cdk import aws_logs as logs from aws_cdk import aws_rds as rds -from aws_cdk import aws_secretsmanager as secretsmanager -from aws_cdk.aws_apigatewayv2_integrations_alpha import HttpLambdaIntegration from config import ( eoAPISettings, eoDBSettings, @@ -22,103 +17,16 @@ eoVectorSettings, ) from constructs import Construct +from eoapi_cdk import ( + PgStacApiLambda, + PgStacDatabase, + TiPgApiLambda, + TitilerPgstacApiLambda, +) eoapi_settings = eoAPISettings() -class BootstrappedDb(Construct): - """ - Given an RDS database, connect to DB and create a database, user, and - password - """ - - def __init__( - self, - scope: Construct, - id: str, - db: rds.DatabaseInstance, - new_dbname: str, - new_username: str, - secrets_prefix: str, - pgstac_version: str, - enable_context: bool = False, - enable_mosaic_index: bool = False, - context_dir: str = "../../", - ) -> None: - """Update RDS database.""" - super().__init__(scope, id) - - # TODO: Utilize a singleton function. - handler = aws_lambda.Function( - self, - "DatabaseBootstrapper", - handler="handler.handler", - runtime=aws_lambda.Runtime.PYTHON_3_10, - code=aws_lambda.Code.from_docker_build( - path=os.path.abspath(context_dir), - file="infrastructure/aws/dockerfiles/Dockerfile.db", - build_args={"PYTHON_VERSION": "3.10", "PGSTAC_VERSION": pgstac_version}, - platform="linux/amd64", - ), - timeout=Duration.minutes(5), - vpc=db.vpc, - allow_public_subnet=True, - log_retention=logs.RetentionDays.ONE_WEEK, - ) - - self.secret = secretsmanager.Secret( - self, - id, - secret_name=os.path.join( - secrets_prefix, id.replace(" ", "_"), self.node.addr - ), - generate_secret_string=secretsmanager.SecretStringGenerator( - secret_string_template=json.dumps( - { - "dbname": new_dbname, - "engine": "postgres", - "port": 5432, - "host": db.instance_endpoint.hostname, - "username": new_username, - }, - ), - generate_string_key="password", - exclude_punctuation=True, - ), - description=f"Deployed by {Stack.of(self).stack_name}", - ) - - self.resource = CustomResource( - scope=scope, - id="BootstrappedDbResource", - service_token=handler.function_arn, - properties={ - # By setting pgstac_version in the properties assures - # that Create/Update events will be passed to the service token - "pgstac_version": pgstac_version, - "context": enable_context, - "mosaic_index": enable_mosaic_index, - "conn_secret_arn": db.secret.secret_arn, - "new_user_secret_arn": self.secret.secret_arn, - }, - # We do not need to run the custom resource on STAC Delete - # Custom Resource are not physical resources so it's OK to `Retain` it - removal_policy=RemovalPolicy.RETAIN, - ) - - # Allow lambda to... - # read new user secret - self.secret.grant_read(handler) - # read database secret - db.secret.grant_read(handler) - # connect to database - db.connections.allow_from(handler, port_range=ec2.Port.tcp(5432)) - - def is_required_by(self, construct: Construct): - """Register required services.""" - return construct.node.add_dependency(self.resource) - - class eoAPIconstruct(Stack): """Earth Observation API CDK application""" @@ -181,62 +89,76 @@ def __init__( # noqa: C901 vpc.add_gateway_endpoint(key, service=service) eodb_settings = eoDBSettings() - db = rds.DatabaseInstance( + + pgstac_db = PgStacDatabase( self, - f"{id}-postgres-db", + "pgstac-db", vpc=vpc, - engine=rds.DatabaseInstanceEngine.POSTGRES, + engine=rds.DatabaseInstanceEngine.postgres( + version=rds.PostgresEngineVersion.VER_14 + ), + vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC), + allocated_storage=eodb_settings.allocated_storage, instance_type=ec2.InstanceType.of( ec2.InstanceClass.BURSTABLE3, ec2.InstanceSize(eodb_settings.instance_size), ), database_name="postgres", - # should set the subnet to `PRIVATE_ISOLATED` but then we need either a bastion host to connect to the db - # or an API to ingest/delete data in the DB - vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC), backup_retention=Duration.days(7), deletion_protection=eoapi_settings.stage.lower() == "production", removal_policy=RemovalPolicy.SNAPSHOT if eoapi_settings.stage.lower() == "production" else RemovalPolicy.DESTROY, - ) - - setup_db = BootstrappedDb( - self, - "STAC DB for eoapi", - db=db, - new_dbname=eodb_settings.dbname, - new_username=eodb_settings.user, + custom_resource_properties={ + "pgstac_version": eodb_settings.pgstac_version, + "context": eodb_settings.context, + "mosaic_index": eodb_settings.mosaic_index, + }, + bootstrapper_lambda_function_options={ + "handler": "handler.handler", + "runtime": aws_lambda.Runtime.PYTHON_3_10, + "code": aws_lambda.Code.from_docker_build( + path=os.path.abspath(context_dir), + file="infrastructure/aws/dockerfiles/Dockerfile.db", + build_args={ + "PYTHON_VERSION": "3.10", + "PGSTAC_VERSION": eodb_settings.pgstac_version, + }, + platform="linux/amd64", + ), + "timeout": Duration.minutes(5), + "allow_public_subnet": True, + "log_retention": logs.RetentionDays.ONE_WEEK, + }, + pgstac_db_name=eodb_settings.dbname, + pgstac_username=eodb_settings.user, secrets_prefix=os.path.join(stage, name), - pgstac_version=eodb_settings.pgstac_version, - enable_context=eodb_settings.context, - enable_mosaic_index=eodb_settings.mosaic_index, - context_dir=context_dir, ) CfnOutput( self, f"{id}-database-secret-arn", - value=db.secret.secret_arn, + value=pgstac_db.pgstac_secret.secret_arn, description="Arn of the SecretsManager instance holding the connection info for Postgres DB", ) # eoapi.raster if "raster" in eoapi_settings.functions: + db_secrets = { - "POSTGRES_HOST": setup_db.secret.secret_value_from_json( + "POSTGRES_HOST": pgstac_db.pgstac_secret.secret_value_from_json( "host" ).to_string(), - "POSTGRES_DBNAME": setup_db.secret.secret_value_from_json( + "POSTGRES_DBNAME": pgstac_db.pgstac_secret.secret_value_from_json( "dbname" ).to_string(), - "POSTGRES_USER": setup_db.secret.secret_value_from_json( + "POSTGRES_USER": pgstac_db.pgstac_secret.secret_value_from_json( "username" ).to_string(), - "POSTGRES_PASS": setup_db.secret.secret_value_from_json( + "POSTGRES_PASS": pgstac_db.pgstac_secret.secret_value_from_json( "password" ).to_string(), - "POSTGRES_PORT": setup_db.secret.secret_value_from_json( + "POSTGRES_PORT": pgstac_db.pgstac_secret.secret_value_from_json( "port" ).to_string(), } @@ -245,76 +167,56 @@ def __init__( # noqa: C901 env = eoraster_settings.env or {} if "DB_MAX_CONN_SIZE" not in env: env["DB_MAX_CONN_SIZE"] = "1" + env.update(db_secrets) - eoraster_function = aws_lambda.Function( + eoraster = TitilerPgstacApiLambda( self, f"{id}-raster-lambda", - runtime=aws_lambda.Runtime.PYTHON_3_11, - code=aws_lambda.Code.from_docker_build( - path=os.path.abspath(context_dir), - file="infrastructure/aws/dockerfiles/Dockerfile.raster", - build_args={ - "PYTHON_VERSION": "3.11", - }, - platform="linux/amd64", - ), + db=pgstac_db.db, + db_secret=pgstac_db.pgstac_secret, vpc=vpc, - vpc_subnets=ec2.SubnetSelection( + subnet_selection=ec2.SubnetSelection( subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS ), - allow_public_subnet=True, - handler="handler.handler", - memory_size=eoraster_settings.memory, - timeout=Duration.seconds(eoraster_settings.timeout), - environment=env, - log_retention=logs.RetentionDays.ONE_WEEK, - ) - for k, v in db_secrets.items(): - eoraster_function.add_environment(key=k, value=str(v)) - - eoraster_function.add_to_role_policy( - iam.PolicyStatement( - actions=["s3:GetObject"], - resources=[ - f"arn:aws:s3:::{bucket}/{eoraster_settings.key}" - for bucket in eoraster_settings.buckets - ], - ) - ) - - db.connections.allow_from(eoraster_function, port_range=ec2.Port.tcp(5432)) - - raster_api = apigw.HttpApi( - self, - f"{id}-raster-endpoint", - default_integration=HttpLambdaIntegration( - f"{id}-raster-integration", - eoraster_function, - ), + api_env=env, + lambda_function_options={ + "code": aws_lambda.Code.from_docker_build( + path=os.path.abspath(context_dir), + file="infrastructure/aws/dockerfiles/Dockerfile.raster", + build_args={ + "PYTHON_VERSION": "3.11", + }, + platform="linux/amd64", + ), + "allow_public_subnet": True, + "handler": "handler.handler", + "runtime": aws_lambda.Runtime.PYTHON_3_11, + "memory_size": eoraster_settings.memory, + "timeout": Duration.seconds(eoraster_settings.timeout), + "log_retention": logs.RetentionDays.ONE_WEEK, + }, + buckets=eoraster_settings.buckets, ) - CfnOutput(self, "eoAPI-raster", value=raster_api.url.strip("/")) - - setup_db.is_required_by(eoraster_function) # eoapi.stac if "stac" in eoapi_settings.functions: db_secrets = { - "POSTGRES_HOST_READER": setup_db.secret.secret_value_from_json( + "POSTGRES_HOST_READER": pgstac_db.pgstac_secret.secret_value_from_json( "host" ).to_string(), - "POSTGRES_HOST_WRITER": setup_db.secret.secret_value_from_json( + "POSTGRES_HOST_WRITER": pgstac_db.pgstac_secret.secret_value_from_json( "host" ).to_string(), - "POSTGRES_DBNAME": setup_db.secret.secret_value_from_json( + "POSTGRES_DBNAME": pgstac_db.pgstac_secret.secret_value_from_json( "dbname" ).to_string(), - "POSTGRES_USER": setup_db.secret.secret_value_from_json( + "POSTGRES_USER": pgstac_db.pgstac_secret.secret_value_from_json( "username" ).to_string(), - "POSTGRES_PASS": setup_db.secret.secret_value_from_json( + "POSTGRES_PASS": pgstac_db.pgstac_secret.secret_value_from_json( "password" ).to_string(), - "POSTGRES_PORT": setup_db.secret.secret_value_from_json( + "POSTGRES_PORT": pgstac_db.pgstac_secret.secret_value_from_json( "port" ).to_string(), } @@ -325,65 +227,54 @@ def __init__( # noqa: C901 env["DB_MAX_CONN_SIZE"] = "1" if "DB_MIN_CONN_SIZE" not in env: env["DB_MIN_CONN_SIZE"] = "1" - - eostac_function = aws_lambda.Function( - self, - f"{id}-stac-lambda", - runtime=aws_lambda.Runtime.PYTHON_3_11, - code=aws_lambda.Code.from_docker_build( - path=os.path.abspath(context_dir), - file="infrastructure/aws/dockerfiles/Dockerfile.stac", - build_args={ - "PYTHON_VERSION": "3.11", - }, - platform="linux/amd64", - ), - vpc=vpc, - handler="handler.handler", - memory_size=eostac_settings.memory, - timeout=Duration.seconds(eostac_settings.timeout), - environment=env, - log_retention=logs.RetentionDays.ONE_WEEK, - ) - for k, v in db_secrets.items(): - eostac_function.add_environment(key=k, value=str(v)) - + env.update(db_secrets) # If raster is deployed we had the TITILER_ENDPOINT env to add the Proxy extension if "raster" in eoapi_settings.functions: - eostac_function.add_environment( - key="TITILER_ENDPOINT", value=raster_api.url.strip("/") - ) + env["TITILER_ENDPOINT"] = eoraster.url.strip("/") - db.connections.allow_from(eostac_function, port_range=ec2.Port.tcp(5432)) - - stac_api = apigw.HttpApi( + PgStacApiLambda( self, - f"{id}-stac-endpoint", - default_integration=HttpLambdaIntegration( - f"{id}-stac-integration", - eostac_function, + id=f"{id}-stac-lambda", + db=pgstac_db.db, + db_secret=pgstac_db.pgstac_secret, + vpc=vpc, + subnet_selection=ec2.SubnetSelection( + subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS ), + api_env=env, + lambda_function_options={ + "runtime": aws_lambda.Runtime.PYTHON_3_11, + "code": aws_lambda.Code.from_docker_build( + path=os.path.abspath(context_dir), + file="infrastructure/aws/dockerfiles/Dockerfile.stac", + build_args={ + "PYTHON_VERSION": "3.11", + }, + platform="linux/amd64", + ), + "handler": "handler.handler", + "memory_size": eostac_settings.memory, + "timeout": Duration.seconds(eostac_settings.timeout), + "log_retention": logs.RetentionDays.ONE_WEEK, + }, ) - CfnOutput(self, "eoAPI-stac", value=stac_api.url.strip("/")) - - setup_db.is_required_by(eostac_function) # eoapi.vector if "vector" in eoapi_settings.functions: db_secrets = { - "POSTGRES_HOST": setup_db.secret.secret_value_from_json( + "POSTGRES_HOST": pgstac_db.pgstac_secret.secret_value_from_json( "host" ).to_string(), - "POSTGRES_DBNAME": setup_db.secret.secret_value_from_json( + "POSTGRES_DBNAME": pgstac_db.pgstac_secret.secret_value_from_json( "dbname" ).to_string(), - "POSTGRES_USER": setup_db.secret.secret_value_from_json( + "POSTGRES_USER": pgstac_db.pgstac_secret.secret_value_from_json( "username" ).to_string(), - "POSTGRES_PASS": setup_db.secret.secret_value_from_json( + "POSTGRES_PASS": pgstac_db.pgstac_secret.secret_value_from_json( "password" ).to_string(), - "POSTGRES_PORT": setup_db.secret.secret_value_from_json( + "POSTGRES_PORT": pgstac_db.pgstac_secret.secret_value_from_json( "port" ).to_string(), } @@ -396,41 +287,34 @@ def __init__( # noqa: C901 if "DB_MIN_CONN_SIZE" not in env: env["DB_MIN_CONN_SIZE"] = "1" - eovector_function = aws_lambda.Function( + env.update(db_secrets) + + TiPgApiLambda( self, f"{id}-vector-lambda", - runtime=aws_lambda.Runtime.PYTHON_3_11, - code=aws_lambda.Code.from_docker_build( - path=os.path.abspath(context_dir), - file="infrastructure/aws/dockerfiles/Dockerfile.vector", - build_args={ - "PYTHON_VERSION": "3.11", - }, - platform="linux/amd64", - ), vpc=vpc, - handler="handler.handler", - memory_size=eovector_settings.memory, - timeout=Duration.seconds(eovector_settings.timeout), - environment=env, - log_retention=logs.RetentionDays.ONE_WEEK, - ) - for k, v in db_secrets.items(): - eovector_function.add_environment(key=k, value=str(v)) - - db.connections.allow_from(eovector_function, port_range=ec2.Port.tcp(5432)) - - vector_api = apigw.HttpApi( - self, - f"{id}-vector-endpoint", - default_integration=HttpLambdaIntegration( - f"{id}-vector-integration", - eovector_function, + db=pgstac_db.db, + db_secret=pgstac_db.pgstac_secret, + subnet_selection=ec2.SubnetSelection( + subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS ), + api_env=env, + lambda_function_options={ + "runtime": aws_lambda.Runtime.PYTHON_3_11, + "code": aws_lambda.Code.from_docker_build( + path=os.path.abspath(context_dir), + file="infrastructure/aws/dockerfiles/Dockerfile.vector", + build_args={ + "PYTHON_VERSION": "3.11", + }, + platform="linux/amd64", + ), + "handler": "handler.handler", + "memory_size": eovector_settings.memory, + "timeout": Duration.seconds(eovector_settings.timeout), + "log_retention": logs.RetentionDays.ONE_WEEK, + }, ) - CfnOutput(self, "eoAPI-vector", value=vector_api.url.strip("/")) - - setup_db.is_required_by(eovector_function) app = App() diff --git a/infrastructure/aws/cdk/config.py b/infrastructure/aws/cdk/config.py index 0450bf9..80d8bba 100644 --- a/infrastructure/aws/cdk/config.py +++ b/infrastructure/aws/cdk/config.py @@ -42,7 +42,7 @@ class eoDBSettings(BaseSettings): instance_size: str = "SMALL" context: bool = True mosaic_index: bool = True - + allocated_storage: int = 20 model_config = { "env_prefix": "CDK_EOAPI_DB_", "env_file": ".env", @@ -90,9 +90,6 @@ class eoRasterSettings(BaseSettings): # ref: https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-arn-format.html buckets: List = ["*"] - # S3 key pattern to limit the access to specific items (e.g: "my_data/*.tif") - key: str = "*" - timeout: int = 10 memory: int = 3008 diff --git a/infrastructure/aws/requirements-cdk.txt b/infrastructure/aws/requirements-cdk.txt index c134c87..404e9f1 100644 --- a/infrastructure/aws/requirements-cdk.txt +++ b/infrastructure/aws/requirements-cdk.txt @@ -1,9 +1,11 @@ # aws cdk -aws-cdk-lib==2.94.0 -aws_cdk-aws_apigatewayv2_alpha==2.94.0a0 -aws_cdk-aws_apigatewayv2_integrations_alpha==2.94.0a0 +aws-cdk-lib==2.99.1 +aws_cdk-aws_apigatewayv2_alpha==2.99.1a0 +aws_cdk-aws_apigatewayv2_integrations_alpha==2.99.1a0 constructs>=10.0.0 # pydantic settings pydantic~=2.0 pydantic-settings~=2.0 + +../eoapi-cdk/dist/python/eoapi_cdk-5.4.0-py3-none-any.whl \ No newline at end of file