feat: Support installing poetry dependencies with pip (#311)

Co-authored-by: Anton Babenko <[email protected]>

.github/workflows/test.yml

@@ -0,0 +1,37 @@
+name: Tests
+
+env:
+  PYTEST_VERSION: 7.1.3
+
+on:
+  push:
+    branches: [master]
+    tags: ["*"]
+  pull_request:
+    branches: [master]
+
+jobs:
+  tests:
+    name: Test with Python ${{ matrix.python_version }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python_version: ["3.7", "3.8", "3.9", "3.10"]
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python_version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python_version }}
+
+      - name: Install poetry and tox
+        shell: bash
+        run: |
+          pip install pytest==${PYTEST_VERSION}
+
+      - name: Run tox
+        shell: bash
+        run: |
+          python -m pytest -vvv tests/

.gitignore

@@ -32,3 +32,6 @@ terraform.rc
 # Lambda directories
 builds/
 __pycache__/
+
+# Test directories
+.tox

README.md

@@ -819,6 +819,29 @@ No modules.
 | <a name="output_s3_object"></a> [s3\_object](#output\_s3\_object) | The map with S3 object data of zip archive deployed (if deployment was from S3) |
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 
+## Development
+
+### Python
+
+During development involving modifying python files, use tox to run unit tests:
+
+```
+tox
+```
+
+This will try to run unit tests which each supported python version, reporting errors for python versions which are not installed locally.
+
+If you only want to test against your main python version:
+
+```
+tox -e py
+```
+
+You can also pass additional positional arguments to pytest which is used to run test, e.g. to make it verbose:
+```
+tox -e py -- -vvv
+```
+
 ## Authors
 
 Module managed by [Anton Babenko](https://github.com/antonbabenko). Check out [serverless.tf](https://serverless.tf) to learn more about doing serverless with Terraform.

examples/build-package/README.md

@@ -36,8 +36,10 @@ Note that this example may create resources which cost money. Run `terraform des
 | <a name="module_lambda_function_from_package"></a> [lambda\_function\_from\_package](#module\_lambda\_function\_from\_package) | ../../ | n/a |
 | <a name="module_lambda_layer"></a> [lambda\_layer](#module\_lambda\_layer) | ../../ | n/a |
 | <a name="module_lambda_layer_pip_requirements"></a> [lambda\_layer\_pip\_requirements](#module\_lambda\_layer\_pip\_requirements) | ../.. | n/a |
+| <a name="module_lambda_layer_poetry"></a> [lambda\_layer\_poetry](#module\_lambda\_layer\_poetry) | ../../ | n/a |
 | <a name="module_package_dir"></a> [package\_dir](#module\_package\_dir) | ../../ | n/a |
 | <a name="module_package_dir_pip_dir"></a> [package\_dir\_pip\_dir](#module\_package\_dir\_pip\_dir) | ../../ | n/a |
+| <a name="module_package_dir_poetry"></a> [package\_dir\_poetry](#module\_package\_dir\_poetry) | ../../ | n/a |
 | <a name="module_package_dir_with_npm_install"></a> [package\_dir\_with\_npm\_install](#module\_package\_dir\_with\_npm\_install) | ../../ | n/a |
 | <a name="module_package_dir_without_npm_install"></a> [package\_dir\_without\_npm\_install](#module\_package\_dir\_without\_npm\_install) | ../../ | n/a |
 | <a name="module_package_dir_without_pip_install"></a> [package\_dir\_without\_pip\_install](#module\_package\_dir\_without\_pip\_install) | ../../ | n/a |

examples/build-package/main.tf

@@ -17,28 +17,52 @@ resource "random_pet" "this" {
 # Build packages
 #################
 
-# Create zip-archive of a single directory where "pip install" will also be executed (default for python runtime)
+# Create zip-archive of a single directory where "pip install" will also be executed (default for python runtime with requirements.txt present)
 module "package_dir" {
   source = "../../"
 
   create_function = false
 
-  runtime     = "python3.8"
-  source_path = "${path.module}/../fixtures/python3.8-app1"
+  build_in_docker = true
+  runtime         = "python3.8"
+  source_path     = "${path.module}/../fixtures/python3.8-app1"
+  artifacts_dir   = "${path.root}/builds/package_dir/"
 }
 
-# Create zip-archive of a single directory where "pip install" will also be executed (default for python runtime) and set temporary directory for pip install
+# Create zip-archive of a single directory where "pip install" will also be executed (default for python runtime with requirements.txt present) and set temporary directory for pip install
 module "package_dir_pip_dir" {
   source = "../../"
 
   create_function = false
 
-  runtime = "python3.8"
+  build_in_docker = true
+  runtime         = "python3.8"
   source_path = [{
     path             = "${path.module}/../fixtures/python3.8-app1"
     pip_tmp_dir      = "${path.cwd}/../fixtures"
     pip_requirements = "${path.module}/../fixtures/python3.8-app1/requirements.txt"
   }]
+  artifacts_dir = "${path.root}/builds/package_dir_pip_dir/"
+}
+
+# Create zip-archive of a single directory where "poetry export" & "pip install --no-deps" will also be executed
+module "package_dir_poetry" {
+  source = "../../"
+
+  create_function = false
+
+  build_in_docker = true
+  runtime         = "python3.9"
+  docker_image    = "build-python3.9-poetry"
+  docker_file     = "${path.module}/../fixtures/python3.9-app-poetry/docker/Dockerfile"
+
+  source_path = [
+    {
+      path           = "${path.module}/../fixtures/python3.9-app-poetry"
+      poetry_install = true
+    }
+  ]
+  artifacts_dir = "${path.root}/builds/package_dir_poetry/"
 }
 
 # Create zip-archive of a single directory without running "pip install" (which is default for python runtime)
@@ -280,6 +304,29 @@ module "lambda_layer" {
   runtime         = "python3.8"
   docker_image    = "public.ecr.aws/sam/build-python3.8:latest"
   docker_file     = "${path.module}/../fixtures/python3.8-app1/docker/Dockerfile"
+  artifacts_dir   = "${path.root}/builds/lambda_layer/"
+}
+
+module "lambda_layer_poetry" {
+  source = "../../"
+
+  create_layer        = true
+  layer_name          = "${random_pet.this.id}-layer-poetry-dockerfile"
+  compatible_runtimes = ["python3.9"]
+
+  source_path = [
+    {
+      path           = "${path.module}/../fixtures/python3.9-app-poetry"
+      poetry_install = true
+    }
+  ]
+  hash_extra = "extra-hash-to-prevent-conflicts-with-module.package_dir"
+
+  build_in_docker = true
+  runtime         = "python3.9"
+  docker_image    = "build-python3.9-poetry"
+  docker_file     = "${path.module}/../fixtures/python3.9-app-poetry/docker/Dockerfile"
+  artifacts_dir   = "${path.root}/builds/lambda_layer_poetry/"
 }
 
 #######################

examples/fixtures/python3.8-app1/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM lambci/lambda:build-python3.8 as build
+FROM public.ecr.aws/sam/build-python3.8 as build
 
 LABEL maintainer="Betajob AS" \
       description="Patched AWS Lambda build container"
@@ -20,7 +20,7 @@ RUN \
  && rpmbuild -ba SPECS/automake.spec --nocheck \
  && yum install -y RPMS/noarch/*
 
-FROM lambci/lambda:build-python3.8
+FROM public.ecr.aws/sam/build-python3.8
 COPY --from=build /root/rpmbuild/RPMS/noarch/*.rpm .
 RUN yum install -y *.rpm \
  && rm *.rpm

examples/fixtures/python3.9-app-poetry/docker/Dockerfile

@@ -0,0 +1,6 @@
+FROM public.ecr.aws/sam/build-python3.9
+
+LABEL maintainer="Betajob AS" \
+      description="Patched AWS Lambda build container"
+
+RUN pip install poetry==1.2.2

examples/fixtures/python3.9-app-poetry/ignore_please.txt

@@ -0,0 +1 @@
+This file should not be included in archive.

examples/fixtures/python3.9-app-poetry/index.py

@@ -0,0 +1,4 @@
+def lambda_handler(event, context):
+    print("Hello from app1!")
+
+    return event

examples/fixtures/python3.9-app-poetry/pyproject.toml

@@ -0,0 +1,15 @@
+[tool.poetry]
+name = "python3.9-app-poetry"
+version = "0.1.0"
+description = ""
+authors = ["Your Name <[email protected]>"]
+
+[tool.poetry.dependencies]
+python = "^3.7"
+colorful = "^0.5.4"
+
+[tool.poetry.dev-dependencies]
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"

examples/simple/main.tf

@@ -301,7 +301,7 @@ module "lambda_function" {
   #  docker_with_ssh_agent = true
   #  docker_file = "${path.module}/../fixtures/python3.8-app1/docker/Dockerfile"
   #  docker_build_root = "${path.module}/../../docker"
-  #  docker_image = "lambci/lambda:build-python3.8"
+  #  docker_image = "public.ecr.aws/sam/build-python3.8"
 }
 
 ####