diff --git a/api/terraform/lambda_layer_genai.tf b/api/terraform/lambda_layer_genai.tf index 66757f1c..53b66586 100644 --- a/api/terraform/lambda_layer_genai.tf +++ b/api/terraform/lambda_layer_genai.tf @@ -8,55 +8,55 @@ # that includes the packages required by OpenAI API and LangChain. #------------------------------------------------------------------------------ locals { - layer_slug = "genai" - layer_name = "layer_${local.layer_slug}" - layer_parent_directory = "${path.module}/python" - layer_source_directory = "${local.layer_parent_directory}/${local.layer_name}" - layer_packaging_script = "${local.layer_source_directory}/create_container.sh" - layer_package_folder = local.layer_slug - layer_dist_build_path = "${path.module}/build/" - layer_dist_package_name = "${local.layer_name}_dst.zip" + genai_layer_slug = "genai" + genai_layer_name = "layer_${local.genai_layer_slug}" + genai_layer_parent_directory = "${path.module}/python" + genai_layer_source_directory = "${local.genai_layer_parent_directory}/${local.genai_layer_name}" + genai_layer_packaging_script = "${local.genai_layer_source_directory}/create_container.sh" + genai_layer_package_folder = local.genai_layer_slug + genai_layer_dist_build_path = "${path.module}/build/" + genai_layer_dist_package_name = "${local.genai_layer_name}_dst.zip" } ############################################################################### # Python package # https://alek-cora-glez.medium.com/deploying-aws-lambda-function-with-terraform-custom-dependencies-7874407cd4fc ############################################################################### -resource "null_resource" "package_layer_genai" { +resource "null_resource" "package_genai_layer" { triggers = { redeployment = sha1(jsonencode([ "${path.module}/lambda_layer.tf", - file("${local.layer_packaging_script}"), - file("${local.layer_source_directory}/Dockerfile"), - file("${local.layer_source_directory}/create_container.sh"), - file("${local.layer_source_directory}/requirements.txt"), - fileexists("${local.layer_source_directory}/${local.layer_dist_package_name}") ? filebase64("${local.layer_source_directory}/${local.layer_dist_package_name}") : null + file("${local.genai_layer_packaging_script}"), + file("${local.genai_layer_source_directory}/Dockerfile"), + file("${local.genai_layer_source_directory}/create_container.sh"), + file("${local.genai_layer_source_directory}/requirements.txt"), + fileexists("${local.genai_layer_source_directory}/${local.genai_layer_dist_package_name}") ? filebase64("${local.genai_layer_source_directory}/${local.genai_layer_dist_package_name}") : null ])) } provisioner "local-exec" { interpreter = ["/bin/bash"] - command = local.layer_packaging_script + command = local.genai_layer_packaging_script environment = { - SOURCE_CODE_PATH = local.layer_source_directory + SOURCE_CODE_PATH = local.genai_layer_source_directory RUNTIME = var.lambda_python_runtime - CONTAINER_NAME = local.layer_name - PACKAGE_NAME = local.layer_dist_package_name + CONTAINER_NAME = local.genai_layer_name + PACKAGE_NAME = local.genai_layer_dist_package_name } } } resource "aws_lambda_layer_version" "genai" { - filename = "${local.layer_source_directory}/${local.layer_dist_package_name}" - source_code_hash = fileexists("${local.layer_source_directory}/${local.layer_dist_package_name}") ? filebase64sha256("${local.layer_source_directory}/${local.layer_dist_package_name}") : null - layer_name = local.layer_slug + filename = "${local.genai_layer_source_directory}/${local.genai_layer_dist_package_name}" + source_code_hash = fileexists("${local.genai_layer_source_directory}/${local.genai_layer_dist_package_name}") ? filebase64sha256("${local.genai_layer_source_directory}/${local.genai_layer_dist_package_name}") : null + layer_name = local.genai_layer_slug compatible_architectures = var.compatible_architectures compatible_runtimes = [var.lambda_python_runtime] lifecycle { create_before_destroy = true } depends_on = [ - null_resource.package_layer_genai + null_resource.package_genai_layer ] } diff --git a/api/terraform/lambda_layer_nlp.tf b/api/terraform/lambda_layer_nlp.tf new file mode 100644 index 00000000..be686273 --- /dev/null +++ b/api/terraform/lambda_layer_nlp.tf @@ -0,0 +1,62 @@ +#------------------------------------------------------------------------------ +# written by: Lawrence McDaniel +# https://lawrencemcdaniel.com/ +# +# date: sep-2023 +# +# usage: implement a Python Lambda layer with a virtual environment +# that includes the packages required by OpenAI API and LangChain. +#------------------------------------------------------------------------------ +locals { + nlp_layer_slug = "nlp" + nlp_layer_name = "layer_${local.nlp_layer_slug}" + nlp_layer_parent_directory = "${path.module}/python" + nlp_layer_source_directory = "${local.nlp_layer_parent_directory}/${local.nlp_layer_name}" + nlp_layer_packaging_script = "${local.nlp_layer_source_directory}/create_container.sh" + nlp_layer_package_folder = local.nlp_layer_slug + nlp_layer_dist_build_path = "${path.module}/build/" + nlp_layer_dist_package_name = "${local.nlp_layer_name}_dst.zip" +} + +############################################################################### +# Python package +# https://alek-cora-glez.medium.com/deploying-aws-lambda-function-with-terraform-custom-dependencies-7874407cd4fc +############################################################################### +resource "null_resource" "package_nlp_layer" { + triggers = { + redeployment = sha1(jsonencode([ + "${path.module}/lambda_layer.tf", + file("${local.nlp_layer_packaging_script}"), + file("${local.nlp_layer_source_directory}/Dockerfile"), + file("${local.nlp_layer_source_directory}/create_container.sh"), + file("${local.nlp_layer_source_directory}/requirements.txt"), + fileexists("${local.nlp_layer_source_directory}/${local.nlp_layer_dist_package_name}") ? filebase64("${local.nlp_layer_source_directory}/${local.nlp_layer_dist_package_name}") : null + ])) + } + + provisioner "local-exec" { + interpreter = ["/bin/bash"] + command = local.nlp_layer_packaging_script + + environment = { + SOURCE_CODE_PATH = local.nlp_layer_source_directory + RUNTIME = var.lambda_python_runtime + CONTAINER_NAME = local.nlp_layer_name + PACKAGE_NAME = local.nlp_layer_dist_package_name + } + } +} + +resource "aws_lambda_layer_version" "nlp" { + filename = "${local.nlp_layer_source_directory}/${local.nlp_layer_dist_package_name}" + source_code_hash = fileexists("${local.nlp_layer_source_directory}/${local.nlp_layer_dist_package_name}") ? filebase64sha256("${local.nlp_layer_source_directory}/${local.nlp_layer_dist_package_name}") : null + layer_name = local.nlp_layer_slug + compatible_architectures = var.compatible_architectures + compatible_runtimes = [var.lambda_python_runtime] + lifecycle { + create_before_destroy = true + } + depends_on = [ + null_resource.package_nlp_layer + ] +} diff --git a/api/terraform/lambda_openai_function.tf b/api/terraform/lambda_openai_function.tf index 2a94e5bc..c6108a20 100644 --- a/api/terraform/lambda_openai_function.tf +++ b/api/terraform/lambda_openai_function.tf @@ -79,7 +79,7 @@ resource "aws_lambda_function" "lambda_openai_function" { architectures = var.compatible_architectures filename = data.archive_file.lambda_openai_function.output_path source_code_hash = data.archive_file.lambda_openai_function.output_base64sha256 - layers = [aws_lambda_layer_version.genai.arn] + layers = [aws_lambda_layer_version.genai.arn, aws_lambda_layer_version.nlp.arn] tags = var.tags environment { diff --git a/api/terraform/python/layer_genai/Dockerfile b/api/terraform/python/layer_genai/Dockerfile index 0e763bf8..5ae335a2 100644 --- a/api/terraform/python/layer_genai/Dockerfile +++ b/api/terraform/python/layer_genai/Dockerfile @@ -9,8 +9,5 @@ WORKDIR /var/task COPY requirements.txt . -ENV PYTHONPATH "${PYTHONPATH}:python/lib/python3.11/site-packages" - RUN yum install -y zip RUN pip install -r requirements.txt --target python/lib/python3.11/site-packages -RUN python3 -m spacy download en_core_web_sm diff --git a/api/terraform/python/layer_genai/requirements.txt b/api/terraform/python/layer_genai/requirements.txt index 0cbfc26c..b24ba475 100644 --- a/api/terraform/python/layer_genai/requirements.txt +++ b/api/terraform/python/layer_genai/requirements.txt @@ -21,9 +21,3 @@ pydantic==2.5.3 pydantic-settings==2.1.0 python-dotenv==1.0.0 python-hcl2==4.3.2 - -# NLP requirements -# -------------------------- -python-Levenshtein==0.23.0 -spacy==3.7.2 -pyyaml diff --git a/api/terraform/python/layer_nlp/.gitignore b/api/terraform/python/layer_nlp/.gitignore new file mode 100644 index 00000000..5e3397e7 --- /dev/null +++ b/api/terraform/python/layer_nlp/.gitignore @@ -0,0 +1,3 @@ +*.zip +venv +archive diff --git a/api/terraform/python/layer_nlp/Dockerfile b/api/terraform/python/layer_nlp/Dockerfile new file mode 100644 index 00000000..0e763bf8 --- /dev/null +++ b/api/terraform/python/layer_nlp/Dockerfile @@ -0,0 +1,16 @@ +# Use an AWS Lambda Python runtime as the base image +# https://hub.docker.com/r/amazon/aws-lambda-python +# ------------------------------------------------------ + +# Stage 1: Build dependencies for x86_64 +FROM --platform=linux/amd64 public.ecr.aws/lambda/python:3.11 + +WORKDIR /var/task + +COPY requirements.txt . + +ENV PYTHONPATH "${PYTHONPATH}:python/lib/python3.11/site-packages" + +RUN yum install -y zip +RUN pip install -r requirements.txt --target python/lib/python3.11/site-packages +RUN python3 -m spacy download en_core_web_sm diff --git a/api/terraform/python/layer_nlp/README.md b/api/terraform/python/layer_nlp/README.md new file mode 100644 index 00000000..30b5a613 --- /dev/null +++ b/api/terraform/python/layer_nlp/README.md @@ -0,0 +1,3 @@ +# AWS Lambda Layer for OpenAI/Langchain Lambdas + +This layer contains the combined pip requirements for Natural Language Processing features. diff --git a/api/terraform/python/layer_nlp/create_container.sh b/api/terraform/python/layer_nlp/create_container.sh new file mode 100755 index 00000000..b35f23d7 --- /dev/null +++ b/api/terraform/python/layer_nlp/create_container.sh @@ -0,0 +1,26 @@ +#!/bin/bash +#------------------------------------------------------------------------------ +# written by: Lawrence McDaniel +# https://lawrencemcdaniel.com/ +# +# date: nov-2023 +# +# usage: Lambda Python packaging tool. +# Called by Terraform "null_resource". Copies python +# module(s) plus any requirements to a dedicated folder so that +# it can be archived to a zip file for upload to +# AWS Lambda by Terraform. +#------------------------------------------------------------------------------ +cd $SOURCE_CODE_PATH + + +docker build -t $CONTAINER_NAME . +docker rm $CONTAINER_NAME +docker run --name $CONTAINER_NAME --entrypoint /bin/bash $CONTAINER_NAME -c "zip -r $PACKAGE_NAME ." + +# Delete the distribution package if it exists +if [ -f $PACKAGE_NAME ]; then + rm $PACKAGE_NAME +fi + +docker cp $CONTAINER_NAME:/var/task/$PACKAGE_NAME . diff --git a/api/terraform/python/layer_nlp/requirements.txt b/api/terraform/python/layer_nlp/requirements.txt new file mode 100644 index 00000000..2a344005 --- /dev/null +++ b/api/terraform/python/layer_nlp/requirements.txt @@ -0,0 +1,15 @@ +# ----------------------------------------------------------------------------- +# written by: Lawrence McDaniel +# https://lawrencemcdaniel.com +# +# usage: Shared Python requirements for AWS Lambda functions. +# Create a virtual environment in the root of this repository +# named `venv`. Terraform modules will look for and include these +# requirements in the zip package for this layer. +# ----------------------------------------------------------------------------- + +# NLP requirements +# -------------------------- +python-Levenshtein==0.23.0 +spacy==3.7.2 +pyyaml