From 56c7d47f4ca9f0576fbeb5ac8d2796f9c254bf4f Mon Sep 17 00:00:00 2001 From: Asen Alexandrov Date: Tue, 17 Jan 2023 09:21:08 +0200 Subject: [PATCH 1/8] feat: Add Makefile and python-builder Dockerfile --- Makefile | 4 ++++ python/.dockerignore | 2 ++ python/Dockerfile | 11 +++++++++++ python/Makefile | 18 ++++++++++++++++++ python/v3.11.1/wl-build.sh | 2 +- 5 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 python/.dockerignore create mode 100644 python/Dockerfile create mode 100644 python/Makefile diff --git a/Makefile b/Makefile index 8ac40fc..c531a0c 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,10 @@ php/wasmedge-php-7.4.32: ruby/v*: make -C ruby $(subst ruby/,,$@) +.PHONY: python/v* +python/v*: + make -C python $(subst python/,,$@) + .PHONY: clean clean: make -C php clean diff --git a/python/.dockerignore b/python/.dockerignore new file mode 100644 index 0000000..aa64ad5 --- /dev/null +++ b/python/.dockerignore @@ -0,0 +1,2 @@ +build-output +build-staging \ No newline at end of file diff --git a/python/Dockerfile b/python/Dockerfile new file mode 100644 index 0000000..4ed5b52 --- /dev/null +++ b/python/Dockerfile @@ -0,0 +1,11 @@ +ARG WASI_SDK_VERSION=19 +FROM ghcr.io/vmware-labs/wasi-builder:${WASI_SDK_VERSION} + +# If more capabilities are required from the build-erpython, consult this +# github workflow configuration for a list of possible dependencies - +# https://github.com/python/cpython/blob/main/.github/workflows/posix-deps-apt.sh +RUN DEBIAN_FRONTEND=noninteractive apt install -y \ + uuid-dev \ + zlib1g-dev + +ADD . /wlr/php diff --git a/python/Makefile b/python/Makefile new file mode 100644 index 0000000..7a11328 --- /dev/null +++ b/python/Makefile @@ -0,0 +1,18 @@ +WASI_SDK_VERSION ?= 19 + +ROOT_DIR := $(dir $(realpath $(lastword $(MAKEFILE_LIST)))) + +include ../Makefile.builders + +.PHONY: python-builder +python-builder: wasi-builder-19 + docker build -f ${ROOT_DIR}/Dockerfile --build-arg WASI_SDK_VERSION=$(WASI_SDK_VERSION) -t ghcr.io/vmware-labs/python-builder:wasi-$(WASI_SDK_VERSION) ${ROOT_DIR} + +.PHONY: v* +v*: python-builder + mkdir -p build-output build-staging + docker run --rm -e WASMLABS_RUNTIME -v ${ROOT_DIR}/build-output:/wlr/build-output -v ${ROOT_DIR}/build-staging:/wlr/build-staging ghcr.io/vmware-labs/python-builder:wasi-${WASI_SDK_VERSION} ./wl-make.sh python/$@ + +.PHONY: clean +clean: + rm -rf build-output build-staging diff --git a/python/v3.11.1/wl-build.sh b/python/v3.11.1/wl-build.sh index f985ed8..06a690c 100644 --- a/python/v3.11.1/wl-build.sh +++ b/python/v3.11.1/wl-build.sh @@ -52,6 +52,6 @@ mkdir -p ${WASMLABS_OUTPUT}/bin 2>/dev/null || exit 1 mkdir -p ${WASMLABS_OUTPUT}/usr/local/lib 2>/dev/null || exit 1 cp python.wasm ${WASMLABS_OUTPUT}/bin/python${WASMLABS_RUNTIME:+-$WASMLABS_RUNTIME}.wasm || exit 1 -cp usr/local/lib/python3.11.zip ${WASMLABS_OUTPUT}/usr/local/lib/ || exit 1 +cp usr/local/lib/python311.zip ${WASMLABS_OUTPUT}/usr/local/lib/ || exit 1 logStatus "DONE. Artifacts in ${WASMLABS_OUTPUT}" From 20dd187bebd4dae3bca63f6f76dd090e94292a0c Mon Sep 17 00:00:00 2001 From: Asen Alexandrov Date: Tue, 17 Jan 2023 16:05:58 +0200 Subject: [PATCH 2/8] fix: Revert python build to wasi-sdk-16 --- Makefile.builders | 6 +++++- python/Dockerfile | 2 +- python/Makefile | 5 +++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Makefile.builders b/Makefile.builders index 48e1b16..32f5e5a 100644 --- a/Makefile.builders +++ b/Makefile.builders @@ -8,4 +8,8 @@ wasm-base: .PHONY: wasi-builder-19 wasi-builder-19: wasm-base - docker build --build-arg WASM_BASE=$(WASM_BASE_TAG) --build-arg WASI_SDK_VERSION=19 -f ${BUILDER_ROOT_DIR}/Dockerfile.wasi-builder -t ghcr.io/vmware-labs/wasi-builder:19 ${BUILDER_ROOT_DIR} \ No newline at end of file + docker build --build-arg WASM_BASE=$(WASM_BASE_TAG) --build-arg WASI_SDK_VERSION=19 -f ${BUILDER_ROOT_DIR}/Dockerfile.wasi-builder -t ghcr.io/vmware-labs/wasi-builder:19 ${BUILDER_ROOT_DIR} + +.PHONY: wasi-builder-16 +wasi-builder-16: wasm-base + docker build --build-arg WASM_BASE=$(WASM_BASE_TAG) --build-arg WASI_SDK_VERSION=16 -f ${BUILDER_ROOT_DIR}/Dockerfile.wasi-builder -t ghcr.io/vmware-labs/wasi-builder:16 ${BUILDER_ROOT_DIR} diff --git a/python/Dockerfile b/python/Dockerfile index 4ed5b52..4e5c402 100644 --- a/python/Dockerfile +++ b/python/Dockerfile @@ -1,4 +1,4 @@ -ARG WASI_SDK_VERSION=19 +ARG WASI_SDK_VERSION=16 FROM ghcr.io/vmware-labs/wasi-builder:${WASI_SDK_VERSION} # If more capabilities are required from the build-erpython, consult this diff --git a/python/Makefile b/python/Makefile index 7a11328..f551ed4 100644 --- a/python/Makefile +++ b/python/Makefile @@ -1,11 +1,12 @@ -WASI_SDK_VERSION ?= 19 +WASI_SDK_VERSION ?= 16 +# NOTE - the default python build is failing with wasi-sdk-19 ROOT_DIR := $(dir $(realpath $(lastword $(MAKEFILE_LIST)))) include ../Makefile.builders .PHONY: python-builder -python-builder: wasi-builder-19 +python-builder: wasi-builder-16 docker build -f ${ROOT_DIR}/Dockerfile --build-arg WASI_SDK_VERSION=$(WASI_SDK_VERSION) -t ghcr.io/vmware-labs/python-builder:wasi-$(WASI_SDK_VERSION) ${ROOT_DIR} .PHONY: v* From 5d34337a450c9f95bd6f1f4357205ccb2f75d2be Mon Sep 17 00:00:00 2001 From: Asen Alexandrov Date: Tue, 17 Jan 2023 16:06:41 +0200 Subject: [PATCH 3/8] feat: Optimize the python binary --- python/v3.11.1/wl-build.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/v3.11.1/wl-build.sh b/python/v3.11.1/wl-build.sh index 06a690c..cee9c65 100644 --- a/python/v3.11.1/wl-build.sh +++ b/python/v3.11.1/wl-build.sh @@ -51,7 +51,10 @@ logStatus "Preparing artifacts... " mkdir -p ${WASMLABS_OUTPUT}/bin 2>/dev/null || exit 1 mkdir -p ${WASMLABS_OUTPUT}/usr/local/lib 2>/dev/null || exit 1 -cp python.wasm ${WASMLABS_OUTPUT}/bin/python${WASMLABS_RUNTIME:+-$WASMLABS_RUNTIME}.wasm || exit 1 +logStatus "Optimizing python binary..." +wasm-opt -O2 -o ${WASMLABS_OUTPUT}/bin/python${WASMLABS_RUNTIME:+-$WASMLABS_RUNTIME}.wasm python.wasm || exit 1 + +logStatus "Copying standard libs..." cp usr/local/lib/python311.zip ${WASMLABS_OUTPUT}/usr/local/lib/ || exit 1 logStatus "DONE. Artifacts in ${WASMLABS_OUTPUT}" From 1404dc9a87b00dd639c3b84a40aa1d907b18a6c7 Mon Sep 17 00:00:00 2001 From: Asen Alexandrov Date: Tue, 17 Jan 2023 22:57:48 +0200 Subject: [PATCH 4/8] feat: Add GH for build and publish of python --- .github/workflows/build-python.yaml | 34 ++++++++++ .github/workflows/release-python.yaml | 92 +++++++++++++++++++++++++++ 2 files changed, 126 insertions(+) create mode 100644 .github/workflows/build-python.yaml create mode 100644 .github/workflows/release-python.yaml diff --git a/.github/workflows/build-python.yaml b/.github/workflows/build-python.yaml new file mode 100644 index 0000000..9914eb1 --- /dev/null +++ b/.github/workflows/build-python.yaml @@ -0,0 +1,34 @@ +name: Build Python +on: + push: + # By specifying branches explicitly, we avoid this workflow from + # running on tag push. We have a dedicated workflow to be ran when + # a tag is pushed. + branches: + - "*" + pull_request: +jobs: + build-python: + strategy: + fail-fast: false + matrix: + include: + - version: 3.11.1 + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Build Python + run: make python/v${{ matrix.version }} + - name: Rename artifacts + shell: bash + run: | + sudo mv python/build-output/python/v${{ matrix.version }}/bin/python{,-${{ matrix.version }}}.wasm + - name: Upload python-aio-${{ matrix.version }}.zip artifact + uses: actions/upload-artifact@v3 + with: + name: python-aio-${{ matrix.version }}.zip + path: | + python/build-output/python/v${{ matrix.version }}/bin/python-${{ matrix.version }}.wasm + python/build-output/python/v${{ matrix.version }}/usr + if-no-files-found: error diff --git a/.github/workflows/release-python.yaml b/.github/workflows/release-python.yaml new file mode 100644 index 0000000..a91ce60 --- /dev/null +++ b/.github/workflows/release-python.yaml @@ -0,0 +1,92 @@ +# Note that for this workflow to be triggered, the tag needs to be +# created of the form `python/+`, where +# by convention is YYYYMMDD- (short SHA can be calculated +# with `git rev-parse --short HEAD`). An example of a tag following +# the convention that triggers automation would be +# `python/7.3.33+20221123-8dfe8b9`. +name: Release Python +on: + push: + tags: + - python/* +jobs: + release-python: + strategy: + matrix: + include: + - version: 3.11.1 + runs-on: ubuntu-latest + steps: + - name: Checkout repository + # Only run for the Python version specified in the git tag. + # + # This if could be moved to the parent `job` section when it's + # supported by GitHub (https://github.com/community/community/discussions/37883) + if: startsWith(github.event.ref, format('refs/tags/python/{0}+', matrix.version)) + uses: actions/checkout@v3 + - name: Build Python + # Only run for the Python version specified in the git tag. + # + # This if could be moved to the parent `job` section when it's + # supported by GitHub (https://github.com/community/community/discussions/37883) + if: startsWith(github.event.ref, format('refs/tags/python/{0}+', matrix.version)) + run: make python/v${{ matrix.version }} + - name: Rename release artifacts + # Only run for the Python version specified in the git tag. + # + # This if could be moved to the parent `job` section when it's + # supported by GitHub (https://github.com/community/community/discussions/37883) + if: startsWith(github.event.ref, format('refs/tags/python/{0}+', matrix.version)) + shell: bash + run: | + sudo mv python/build-output/python/v${{ matrix.version }}/bin/python{,-${{ matrix.version }}}.wasm + - name: Install zip + uses: montudor/action-zip@v1 + - name: Bundle Python with standard libraries + run: sudo zip -qq -r python-aio-${{ matrix.version }}.zip bin/python-${{ matrix.version }}.wasm usr + working-directory: python/build-output/python/v${{ matrix.version }} + - name: Create release + # Only run for the Python version specified in the git tag. + # + # This if could be moved to the parent `job` section when it's + # supported by GitHub (https://github.com/community/community/discussions/37883) + if: startsWith(github.event.ref, format('refs/tags/python/{0}+', matrix.version)) + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release create --generate-notes ${{ github.ref_name }} || true + - name: Append Python release assets + # Only run for the Python version specified in the git tag. + # + # This if could be moved to the parent `job` section when it's + # supported by GitHub (https://github.com/community/community/discussions/37883) + if: ${{ startsWith(github.event.ref, format('refs/tags/python/{0}+', matrix.version))}} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release upload ${{ github.ref_name }} \ + python/build-output/python/v${{ matrix.version }}/python-aio-${{ matrix.version }}.zip + - name: Generate release assets digests + # Only run for the Python version specified in the git tag. + # + # This if could be moved to the parent `job` section when it's + # supported by GitHub (https://github.com/community/community/discussions/37883) + if: ${{ startsWith(github.event.ref, format('refs/tags/python/{0}+', matrix.version))}} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + for asset in python/build-output/python/v${{ matrix.version }}/python-aio-${{ matrix.version }}.zip; do + sha256sum "$asset" | sudo tee "$asset.sha256sum" > /dev/null + done + - name: Append release assets digests + # Only run for the Python version specified in the git tag. + # + # This if could be moved to the parent `job` section when it's + # supported by GitHub (https://github.com/community/community/discussions/37883) + if: ${{ startsWith(github.event.ref, format('refs/tags/python/{0}+', matrix.version))}} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release upload ${{ github.ref_name }} \ + python/build-output/python/v${{ matrix.version }}/python-aio-${{ matrix.version }}.zip.sha256sum + From eee3fb1d9a642e7f38517e57e14a7f6d05570fb7 Mon Sep 17 00:00:00 2001 From: Asen Dobrinov Alexandrov Date: Wed, 18 Jan 2023 17:45:41 +0200 Subject: [PATCH 5/8] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Rafael Fernández López --- .github/workflows/release-python.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-python.yaml b/.github/workflows/release-python.yaml index a91ce60..e32a3fb 100644 --- a/.github/workflows/release-python.yaml +++ b/.github/workflows/release-python.yaml @@ -3,7 +3,7 @@ # by convention is YYYYMMDD- (short SHA can be calculated # with `git rev-parse --short HEAD`). An example of a tag following # the convention that triggers automation would be -# `python/7.3.33+20221123-8dfe8b9`. +# `python/3.11.1+20221123-8dfe8b9`. name: Release Python on: push: From 11a5c6c377e811a1951b041755916fbc91c5e19f Mon Sep 17 00:00:00 2001 From: Asen Alexandrov Date: Wed, 18 Jan 2023 23:40:51 +0200 Subject: [PATCH 6/8] chore:Remove use of 3rd party GH action --- .github/workflows/release-python.yaml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release-python.yaml b/.github/workflows/release-python.yaml index e32a3fb..3578189 100644 --- a/.github/workflows/release-python.yaml +++ b/.github/workflows/release-python.yaml @@ -40,11 +40,16 @@ jobs: shell: bash run: | sudo mv python/build-output/python/v${{ matrix.version }}/bin/python{,-${{ matrix.version }}}.wasm - - name: Install zip - uses: montudor/action-zip@v1 - name: Bundle Python with standard libraries - run: sudo zip -qq -r python-aio-${{ matrix.version }}.zip bin/python-${{ matrix.version }}.wasm usr - working-directory: python/build-output/python/v${{ matrix.version }} + # Only run for the Python version specified in the git tag. + # + # This if could be moved to the parent `job` section when it's + # supported by GitHub (https://github.com/community/community/discussions/37883) + if: startsWith(github.event.ref, format('refs/tags/python/{0}+', matrix.version)) + run: | + pushd python/build-output/python/v${{ matrix.version }} + sudo zip -qq -r python-aio-${{ matrix.version }}.zip bin/python-${{ matrix.version }}.wasm usr + popd - name: Create release # Only run for the Python version specified in the git tag. # From 576b47dc36a3ffe2f97af178785ff871a9fa7ad5 Mon Sep 17 00:00:00 2001 From: Asen Alexandrov Date: Wed, 18 Jan 2023 23:53:52 +0200 Subject: [PATCH 7/8] feat: Add sqlite3 to the python build --- libs/sqlite/version-3.39.2/wl-build.sh | 2 +- python/Dockerfile | 1 + python/v3.11.1/wl-build-deps.sh | 7 +++++++ python/v3.11.1/wl-build.sh | 14 ++++++++++++-- 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/libs/sqlite/version-3.39.2/wl-build.sh b/libs/sqlite/version-3.39.2/wl-build.sh index a3a06f7..2efc8d3 100755 --- a/libs/sqlite/version-3.39.2/wl-build.sh +++ b/libs/sqlite/version-3.39.2/wl-build.sh @@ -7,7 +7,7 @@ then fi # export CFLAGS_CONFIG="-O3 -g" -export CFLAGS_CONFIG="-O2" +export CFLAGS_CONFIG="-O0" export CFLAGS_WASI="--sysroot=${WASI_SYSROOT} -I./wasmlabs-stubs -D_WASI_EMULATED_MMAN -D_WASI_EMULATED_GETPID -D_WASI_EMULATED_SIGNAL -D_WASI_EMULATED_PROCESS_CLOCKS" export LDFLAGS_WASI="--sysroot=${WASI_SYSROOT} -lwasi-emulated-mman -lwasi-emulated-getpid -lwasi-emulated-signal -lwasi-emulated-process-clocks" diff --git a/python/Dockerfile b/python/Dockerfile index 4e5c402..5f6c5c4 100644 --- a/python/Dockerfile +++ b/python/Dockerfile @@ -5,6 +5,7 @@ FROM ghcr.io/vmware-labs/wasi-builder:${WASI_SDK_VERSION} # github workflow configuration for a list of possible dependencies - # https://github.com/python/cpython/blob/main/.github/workflows/posix-deps-apt.sh RUN DEBIAN_FRONTEND=noninteractive apt install -y \ + tcl \ uuid-dev \ zlib1g-dev diff --git a/python/v3.11.1/wl-build-deps.sh b/python/v3.11.1/wl-build-deps.sh index 0e7b1f3..5432c45 100644 --- a/python/v3.11.1/wl-build-deps.sh +++ b/python/v3.11.1/wl-build-deps.sh @@ -25,3 +25,10 @@ $WASMLABS_MAKE ${WASMLABS_REPO_ROOT}/libs/zlib/v1.2.11 || exit 1 export CFLAGS_DEPENDENCIES="-I${WASMLABS_OUTPUT_BASE}/zlib/v1.2.11/include ${CFLAGS_DEPENDENCIES}" export LDFLAGS_DEPENDENCIES="-L${WASMLABS_OUTPUT_BASE}/zlib/v1.2.11/lib ${LDFLAGS_DEPENDENCIES}" + + +### sqlite3 +$WASMLABS_MAKE ${WASMLABS_REPO_ROOT}/libs/sqlite/version-3.39.2 || exit 1 + +export CFLAGS_DEPENDENCIES="-I${WASMLABS_OUTPUT_BASE}/sqlite/version-3.39.2/include ${CFLAGS_DEPENDENCIES}" +export LDFLAGS_DEPENDENCIES="-L${WASMLABS_OUTPUT_BASE}/sqlite/version-3.39.2/lib ${LDFLAGS_DEPENDENCIES}" diff --git a/python/v3.11.1/wl-build.sh b/python/v3.11.1/wl-build.sh index cee9c65..ef2399b 100644 --- a/python/v3.11.1/wl-build.sh +++ b/python/v3.11.1/wl-build.sh @@ -26,8 +26,7 @@ else make distclean || exit 1 fi -# export CFLAGS_CONFIG="-O3 -g" -export CFLAGS_CONFIG="-O3" +export CFLAGS_CONFIG="-O0" export CFLAGS="${CFLAGS_CONFIG} ${CFLAGS_DEPENDENCIES} ${CFLAGS}" export LDFLAGS="${LDFLAGS_DEPENDENCIES} ${LDFLAGS}" @@ -39,6 +38,15 @@ then export PYTHON_WASM_CONFIGURE=" --with-wasm-runtime=${WASMLABS_RUNTIME} ${PYTHON_WASM_CONFIGURE}" fi +# By exporting WASMLABS_SKIP_WASM_OPT envvar during the build, the +# wasm-opt wrapper in the wasm-base image will be a dummy wrapper that +# is effectively a NOP. +# +# This is due to https://github.com/llvm/llvm-project/issues/55781, so +# that we get to choose which optimization passes are executed after +# the artifacts have been built. +export WASMLABS_SKIP_WASM_OPT=1 + logStatus "Configuring build with '${PYTHON_WASM_CONFIGURE}'... " CONFIG_SITE=./Tools/wasm/config.site-wasm32-wasi ./configure -C --host=wasm32-wasi --build=$(./config.guess) ${PYTHON_WASM_CONFIGURE} || exit 1 @@ -47,6 +55,8 @@ export MAKE_TARGETS='python.wasm wasm_stdlib' logStatus "Building '${MAKE_TARGETS}'... " make -j ${MAKE_TARGETS} || exit 1 +unset WASMLABS_SKIP_WASM_OPT + logStatus "Preparing artifacts... " mkdir -p ${WASMLABS_OUTPUT}/bin 2>/dev/null || exit 1 mkdir -p ${WASMLABS_OUTPUT}/usr/local/lib 2>/dev/null || exit 1 From 7758906ef38dded490f09b9cbdaaa58895226105 Mon Sep 17 00:00:00 2001 From: Asen Alexandrov Date: Thu, 19 Jan 2023 00:15:34 +0200 Subject: [PATCH 8/8] chore: Add the 'if-no-files-found: error' to all uploads in GH actions --- .github/workflows/build-php.yaml | 2 ++ .github/workflows/build-ruby.yaml | 1 + 2 files changed, 3 insertions(+) diff --git a/.github/workflows/build-php.yaml b/.github/workflows/build-php.yaml index 2e4d1b8..b0c0dc8 100644 --- a/.github/workflows/build-php.yaml +++ b/.github/workflows/build-php.yaml @@ -59,8 +59,10 @@ jobs: with: name: php-${{ matrix.version }}${{ matrix.suffix }}.wasm path: php/build-output/php/php-${{ matrix.version }}/bin/php-${{ matrix.version }}${{ matrix.suffix }}.wasm + if-no-files-found: error - name: Upload php-cgi-${{ matrix.version }}${{ matrix.suffix }}.wasm artifact uses: actions/upload-artifact@v3 with: name: php-cgi-${{ matrix.version }}${{ matrix.suffix }}.wasm path: php/build-output/php/php-${{ matrix.version }}/bin/php-cgi-${{ matrix.version }}${{ matrix.suffix }}.wasm + if-no-files-found: error diff --git a/.github/workflows/build-ruby.yaml b/.github/workflows/build-ruby.yaml index b3f1d85..13c078d 100644 --- a/.github/workflows/build-ruby.yaml +++ b/.github/workflows/build-ruby.yaml @@ -30,3 +30,4 @@ jobs: with: name: ruby-${{ matrix.version }}.wasm path: ruby/build-output/ruby/v${{ matrix.target_version }}/bin/ruby-${{ matrix.version }}.wasm + if-no-files-found: error