From 4eb2f3ba0df3c8074e781f709c84fbeea5835856 Mon Sep 17 00:00:00 2001 From: Leon Luttenberger Date: Fri, 15 Dec 2023 14:30:57 -0600 Subject: [PATCH 1/5] add python 3.12 layers --- building/build-lambda-layers.sh | 8 ++++++++ building/lambda/build-docker-images.sh | 7 +++++++ 2 files changed, 15 insertions(+) diff --git a/building/build-lambda-layers.sh b/building/build-lambda-layers.sh index ab1cad8f5..d3c9ea733 100755 --- a/building/build-lambda-layers.sh +++ b/building/build-lambda-layers.sh @@ -45,3 +45,11 @@ docker run \ --rm \ awswrangler-build-py311 \ build-lambda-layer.sh "${VERSION}-py3.11${ARCH_SUFFIX}" "ninja-build" + +# Python 3.12 +docker run \ + --volume "$DIR_NAME":/aws-sdk-pandas/ \ + --workdir /aws-sdk-pandas/building/lambda \ + --rm \ + awswrangler-build-py312 \ + build-lambda-layer.sh "${VERSION}-py3.12${ARCH_SUFFIX}" "ninja-build" diff --git a/building/lambda/build-docker-images.sh b/building/lambda/build-docker-images.sh index a848b1f01..66ae514ad 100755 --- a/building/lambda/build-docker-images.sh +++ b/building/lambda/build-docker-images.sh @@ -34,4 +34,11 @@ docker build \ --build-arg base_image=public.ecr.aws/lambda/python:3.11 \ . +# Python 3.12 +docker build \ + --pull \ + --tag awswrangler-build-py312 \ + --build-arg base_image=public.ecr.aws/lambda/python:3.12 \ + . + rm -rf pyproject.toml poetry.lock From 2f05521d07b58f07dd762c95eb49425756d22094 Mon Sep 17 00:00:00 2001 From: Leon Luttenberger Date: Fri, 15 Dec 2023 14:42:57 -0600 Subject: [PATCH 2/5] test 3.12 --- building/build-lambda-layers.sh | 62 +++++++++++++------------- building/lambda/Dockerfile | 2 +- building/lambda/build-docker-images.sh | 54 +++++++++++----------- 3 files changed, 59 insertions(+), 59 deletions(-) diff --git a/building/build-lambda-layers.sh b/building/build-lambda-layers.sh index d3c9ea733..c274c79ba 100755 --- a/building/build-lambda-layers.sh +++ b/building/build-lambda-layers.sh @@ -14,37 +14,37 @@ pushd lambda # Building all related docker images ./build-docker-images.sh -# Python 3.8 -docker run \ - --volume "$DIR_NAME":/aws-sdk-pandas/ \ - --workdir /aws-sdk-pandas/building/lambda \ - --rm \ - awswrangler-build-py38 \ - build-lambda-layer.sh "${VERSION}-py3.8${ARCH_SUFFIX}" "ninja-build" - -# Python 3.9 -docker run \ - --volume "$DIR_NAME":/aws-sdk-pandas/ \ - --workdir /aws-sdk-pandas/building/lambda \ - --rm \ - awswrangler-build-py39 \ - build-lambda-layer.sh "${VERSION}-py3.9${ARCH_SUFFIX}" "ninja-build" - -# Python 3.10 -docker run \ - --volume "$DIR_NAME":/aws-sdk-pandas/ \ - --workdir /aws-sdk-pandas/building/lambda \ - --rm \ - awswrangler-build-py310 \ - build-lambda-layer.sh "${VERSION}-py3.10${ARCH_SUFFIX}" "ninja-build" - -# Python 3.11 -docker run \ - --volume "$DIR_NAME":/aws-sdk-pandas/ \ - --workdir /aws-sdk-pandas/building/lambda \ - --rm \ - awswrangler-build-py311 \ - build-lambda-layer.sh "${VERSION}-py3.11${ARCH_SUFFIX}" "ninja-build" +# # Python 3.8 +# docker run \ +# --volume "$DIR_NAME":/aws-sdk-pandas/ \ +# --workdir /aws-sdk-pandas/building/lambda \ +# --rm \ +# awswrangler-build-py38 \ +# build-lambda-layer.sh "${VERSION}-py3.8${ARCH_SUFFIX}" "ninja-build" + +# # Python 3.9 +# docker run \ +# --volume "$DIR_NAME":/aws-sdk-pandas/ \ +# --workdir /aws-sdk-pandas/building/lambda \ +# --rm \ +# awswrangler-build-py39 \ +# build-lambda-layer.sh "${VERSION}-py3.9${ARCH_SUFFIX}" "ninja-build" + +# # Python 3.10 +# docker run \ +# --volume "$DIR_NAME":/aws-sdk-pandas/ \ +# --workdir /aws-sdk-pandas/building/lambda \ +# --rm \ +# awswrangler-build-py310 \ +# build-lambda-layer.sh "${VERSION}-py3.10${ARCH_SUFFIX}" "ninja-build" + +# # Python 3.11 +# docker run \ +# --volume "$DIR_NAME":/aws-sdk-pandas/ \ +# --workdir /aws-sdk-pandas/building/lambda \ +# --rm \ +# awswrangler-build-py311 \ +# build-lambda-layer.sh "${VERSION}-py3.11${ARCH_SUFFIX}" "ninja-build" # Python 3.12 docker run \ diff --git a/building/lambda/Dockerfile b/building/lambda/Dockerfile index 570f00fc4..a1a027cce 100644 --- a/building/lambda/Dockerfile +++ b/building/lambda/Dockerfile @@ -3,7 +3,7 @@ ARG python_version=base FROM ${base_image} AS base -RUN yum install -y \ +RUN dnf install -y \ boost-devel \ jemalloc-devel \ libxml2-devel \ diff --git a/building/lambda/build-docker-images.sh b/building/lambda/build-docker-images.sh index 66ae514ad..11e1dcd8c 100755 --- a/building/lambda/build-docker-images.sh +++ b/building/lambda/build-docker-images.sh @@ -6,33 +6,33 @@ cp ../../poetry.lock . export DOCKER_BUILDKIT=1 -# Python 3.8 -docker build \ - --pull \ - --tag awswrangler-build-py38 \ - --build-arg base_image=public.ecr.aws/lambda/python:3.8 \ - . - -# Python 3.9 -docker build \ - --pull \ - --tag awswrangler-build-py39 \ - --build-arg base_image=public.ecr.aws/lambda/python:3.9 \ - . - -# Python 3.10 -docker build \ - --pull \ - --tag awswrangler-build-py310 \ - --build-arg base_image=public.ecr.aws/lambda/python:3.10 \ - . - -# Python 3.11 -docker build \ - --pull \ - --tag awswrangler-build-py311 \ - --build-arg base_image=public.ecr.aws/lambda/python:3.11 \ - . +# # Python 3.8 +# docker build \ +# --pull \ +# --tag awswrangler-build-py38 \ +# --build-arg base_image=public.ecr.aws/lambda/python:3.8 \ +# . + +# # Python 3.9 +# docker build \ +# --pull \ +# --tag awswrangler-build-py39 \ +# --build-arg base_image=public.ecr.aws/lambda/python:3.9 \ +# . + +# # Python 3.10 +# docker build \ +# --pull \ +# --tag awswrangler-build-py310 \ +# --build-arg base_image=public.ecr.aws/lambda/python:3.10 \ +# . + +# # Python 3.11 +# docker build \ +# --pull \ +# --tag awswrangler-build-py311 \ +# --build-arg base_image=public.ecr.aws/lambda/python:3.11 \ +# . # Python 3.12 docker build \ From 020104a317d483e51a3e786bff7261229ebef1c4 Mon Sep 17 00:00:00 2001 From: Leon Luttenberger Date: Fri, 15 Dec 2023 16:20:38 -0600 Subject: [PATCH 3/5] add distutils --- building/lambda/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/building/lambda/Dockerfile b/building/lambda/Dockerfile index a1a027cce..925e68b7f 100644 --- a/building/lambda/Dockerfile +++ b/building/lambda/Dockerfile @@ -23,7 +23,7 @@ WORKDIR /root FROM ${python_version} COPY pyproject.toml poetry.lock ./ -RUN pip3 install --upgrade pip wheel +RUN pip3 install --upgrade pip wheel distutils RUN pip3 install --upgrade urllib3==1.26.16 # temporary to avoid https://github.com/urllib3/urllib3/issues/2168 (TODO remove when the AL2 image updates to support OpenSSL 1.1.1+) RUN pip3 install --upgrade six cython cmake hypothesis poetry RUN poetry config virtualenvs.create false --local && poetry install --no-root --no-dev From f9196fca980ecc2df8998091ac4532e73c71e459 Mon Sep 17 00:00:00 2001 From: Leon Luttenberger Date: Fri, 15 Dec 2023 16:25:06 -0600 Subject: [PATCH 4/5] Revert "add distutils" This reverts commit 020104a317d483e51a3e786bff7261229ebef1c4. --- building/lambda/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/building/lambda/Dockerfile b/building/lambda/Dockerfile index 925e68b7f..a1a027cce 100644 --- a/building/lambda/Dockerfile +++ b/building/lambda/Dockerfile @@ -23,7 +23,7 @@ WORKDIR /root FROM ${python_version} COPY pyproject.toml poetry.lock ./ -RUN pip3 install --upgrade pip wheel distutils +RUN pip3 install --upgrade pip wheel RUN pip3 install --upgrade urllib3==1.26.16 # temporary to avoid https://github.com/urllib3/urllib3/issues/2168 (TODO remove when the AL2 image updates to support OpenSSL 1.1.1+) RUN pip3 install --upgrade six cython cmake hypothesis poetry RUN poetry config virtualenvs.create false --local && poetry install --no-root --no-dev From 305f6e2f33180f807ae815c73ffcf8e5d70c2992 Mon Sep 17 00:00:00 2001 From: Leon Luttenberger Date: Mon, 18 Dec 2023 09:47:09 -0600 Subject: [PATCH 5/5] add alt dockerfile --- building/build-lambda-layers.sh | 62 +++++++++++++------------- building/lambda/Dockerfile | 2 +- building/lambda/Dockerfile.py312 | 33 ++++++++++++++ building/lambda/build-docker-images.sh | 55 ++++++++++++----------- 4 files changed, 93 insertions(+), 59 deletions(-) create mode 100644 building/lambda/Dockerfile.py312 diff --git a/building/build-lambda-layers.sh b/building/build-lambda-layers.sh index c274c79ba..d3c9ea733 100755 --- a/building/build-lambda-layers.sh +++ b/building/build-lambda-layers.sh @@ -14,37 +14,37 @@ pushd lambda # Building all related docker images ./build-docker-images.sh -# # Python 3.8 -# docker run \ -# --volume "$DIR_NAME":/aws-sdk-pandas/ \ -# --workdir /aws-sdk-pandas/building/lambda \ -# --rm \ -# awswrangler-build-py38 \ -# build-lambda-layer.sh "${VERSION}-py3.8${ARCH_SUFFIX}" "ninja-build" - -# # Python 3.9 -# docker run \ -# --volume "$DIR_NAME":/aws-sdk-pandas/ \ -# --workdir /aws-sdk-pandas/building/lambda \ -# --rm \ -# awswrangler-build-py39 \ -# build-lambda-layer.sh "${VERSION}-py3.9${ARCH_SUFFIX}" "ninja-build" - -# # Python 3.10 -# docker run \ -# --volume "$DIR_NAME":/aws-sdk-pandas/ \ -# --workdir /aws-sdk-pandas/building/lambda \ -# --rm \ -# awswrangler-build-py310 \ -# build-lambda-layer.sh "${VERSION}-py3.10${ARCH_SUFFIX}" "ninja-build" - -# # Python 3.11 -# docker run \ -# --volume "$DIR_NAME":/aws-sdk-pandas/ \ -# --workdir /aws-sdk-pandas/building/lambda \ -# --rm \ -# awswrangler-build-py311 \ -# build-lambda-layer.sh "${VERSION}-py3.11${ARCH_SUFFIX}" "ninja-build" +# Python 3.8 +docker run \ + --volume "$DIR_NAME":/aws-sdk-pandas/ \ + --workdir /aws-sdk-pandas/building/lambda \ + --rm \ + awswrangler-build-py38 \ + build-lambda-layer.sh "${VERSION}-py3.8${ARCH_SUFFIX}" "ninja-build" + +# Python 3.9 +docker run \ + --volume "$DIR_NAME":/aws-sdk-pandas/ \ + --workdir /aws-sdk-pandas/building/lambda \ + --rm \ + awswrangler-build-py39 \ + build-lambda-layer.sh "${VERSION}-py3.9${ARCH_SUFFIX}" "ninja-build" + +# Python 3.10 +docker run \ + --volume "$DIR_NAME":/aws-sdk-pandas/ \ + --workdir /aws-sdk-pandas/building/lambda \ + --rm \ + awswrangler-build-py310 \ + build-lambda-layer.sh "${VERSION}-py3.10${ARCH_SUFFIX}" "ninja-build" + +# Python 3.11 +docker run \ + --volume "$DIR_NAME":/aws-sdk-pandas/ \ + --workdir /aws-sdk-pandas/building/lambda \ + --rm \ + awswrangler-build-py311 \ + build-lambda-layer.sh "${VERSION}-py3.11${ARCH_SUFFIX}" "ninja-build" # Python 3.12 docker run \ diff --git a/building/lambda/Dockerfile b/building/lambda/Dockerfile index a1a027cce..570f00fc4 100644 --- a/building/lambda/Dockerfile +++ b/building/lambda/Dockerfile @@ -3,7 +3,7 @@ ARG python_version=base FROM ${base_image} AS base -RUN dnf install -y \ +RUN yum install -y \ boost-devel \ jemalloc-devel \ libxml2-devel \ diff --git a/building/lambda/Dockerfile.py312 b/building/lambda/Dockerfile.py312 new file mode 100644 index 000000000..a1a027cce --- /dev/null +++ b/building/lambda/Dockerfile.py312 @@ -0,0 +1,33 @@ +ARG base_image +ARG python_version=base + +FROM ${base_image} AS base + +RUN dnf install -y \ + boost-devel \ + jemalloc-devel \ + libxml2-devel \ + libxslt-devel \ + bison \ + make \ + gcc \ + gcc-c++ \ + flex \ + autoconf \ + zip \ + git \ + ninja-build + +WORKDIR /root + +FROM ${python_version} +COPY pyproject.toml poetry.lock ./ + +RUN pip3 install --upgrade pip wheel +RUN pip3 install --upgrade urllib3==1.26.16 # temporary to avoid https://github.com/urllib3/urllib3/issues/2168 (TODO remove when the AL2 image updates to support OpenSSL 1.1.1+) +RUN pip3 install --upgrade six cython cmake hypothesis poetry +RUN poetry config virtualenvs.create false --local && poetry install --no-root --no-dev + +RUN rm -f pyproject.toml poetry.lock + +ENTRYPOINT ["/bin/sh"] diff --git a/building/lambda/build-docker-images.sh b/building/lambda/build-docker-images.sh index 11e1dcd8c..3f36148ea 100755 --- a/building/lambda/build-docker-images.sh +++ b/building/lambda/build-docker-images.sh @@ -6,39 +6,40 @@ cp ../../poetry.lock . export DOCKER_BUILDKIT=1 -# # Python 3.8 -# docker build \ -# --pull \ -# --tag awswrangler-build-py38 \ -# --build-arg base_image=public.ecr.aws/lambda/python:3.8 \ -# . - -# # Python 3.9 -# docker build \ -# --pull \ -# --tag awswrangler-build-py39 \ -# --build-arg base_image=public.ecr.aws/lambda/python:3.9 \ -# . - -# # Python 3.10 -# docker build \ -# --pull \ -# --tag awswrangler-build-py310 \ -# --build-arg base_image=public.ecr.aws/lambda/python:3.10 \ -# . - -# # Python 3.11 -# docker build \ -# --pull \ -# --tag awswrangler-build-py311 \ -# --build-arg base_image=public.ecr.aws/lambda/python:3.11 \ -# . +# Python 3.8 +docker build \ + --pull \ + --tag awswrangler-build-py38 \ + --build-arg base_image=public.ecr.aws/lambda/python:3.8 \ + . + +# Python 3.9 +docker build \ + --pull \ + --tag awswrangler-build-py39 \ + --build-arg base_image=public.ecr.aws/lambda/python:3.9 \ + . + +# Python 3.10 +docker build \ + --pull \ + --tag awswrangler-build-py310 \ + --build-arg base_image=public.ecr.aws/lambda/python:3.10 \ + . + +# Python 3.11 +docker build \ + --pull \ + --tag awswrangler-build-py311 \ + --build-arg base_image=public.ecr.aws/lambda/python:3.11 \ + . # Python 3.12 docker build \ --pull \ --tag awswrangler-build-py312 \ --build-arg base_image=public.ecr.aws/lambda/python:3.12 \ + --file Dockerfile.py312 \ . rm -rf pyproject.toml poetry.lock