Merge pull request #35 from vmware-labs/11-python-automation

#11 Add GH actions for build and publish of python

.github/workflows/build-php.yaml

@@ -59,8 +59,10 @@ jobs:
         with:
           name: php-${{ matrix.version }}${{ matrix.suffix }}.wasm
           path: php/build-output/php/php-${{ matrix.version }}/bin/php-${{ matrix.version }}${{ matrix.suffix }}.wasm
+          if-no-files-found: error
       - name: Upload php-cgi-${{ matrix.version }}${{ matrix.suffix }}.wasm artifact
         uses: actions/upload-artifact@v3
         with:
           name: php-cgi-${{ matrix.version }}${{ matrix.suffix }}.wasm
           path: php/build-output/php/php-${{ matrix.version }}/bin/php-cgi-${{ matrix.version }}${{ matrix.suffix }}.wasm
+          if-no-files-found: error

.github/workflows/build-python.yaml

@@ -0,0 +1,34 @@
+name: Build Python
+on:
+  push:
+    # By specifying branches explicitly, we avoid this workflow from
+    # running on tag push. We have a dedicated workflow to be ran when
+    # a tag is pushed.
+    branches:
+      - "*"
+  pull_request:
+jobs:
+  build-python:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - version: 3.11.1
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+      - name: Build Python
+        run: make python/v${{ matrix.version }}
+      - name: Rename artifacts
+        shell: bash
+        run: |
+          sudo mv python/build-output/python/v${{ matrix.version }}/bin/python{,-${{ matrix.version }}}.wasm
+      - name: Upload python-aio-${{ matrix.version }}.zip artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: python-aio-${{ matrix.version }}.zip
+          path: |
+            python/build-output/python/v${{ matrix.version }}/bin/python-${{ matrix.version }}.wasm
+            python/build-output/python/v${{ matrix.version }}/usr
+          if-no-files-found: error

.github/workflows/build-ruby.yaml

@@ -30,3 +30,4 @@ jobs:
         with:
           name: ruby-${{ matrix.version }}.wasm
           path: ruby/build-output/ruby/v${{ matrix.target_version }}/bin/ruby-${{ matrix.version }}.wasm
+          if-no-files-found: error

.github/workflows/release-python.yaml

@@ -0,0 +1,97 @@
+# Note that for this workflow to be triggered, the tag needs to be
+# created of the form `python/<version>+<buildinfo>`, where <buildinfo>
+# by convention is YYYYMMDD-<short-sha> (short SHA can be calculated
+# with `git rev-parse --short HEAD`). An example of a tag following
+# the convention that triggers automation would be
+# `python/3.11.1+20221123-8dfe8b9`.
+name: Release Python
+on:
+  push:
+    tags:
+      - python/*
+jobs:
+  release-python:
+    strategy:
+      matrix:
+        include:
+          - version: 3.11.1
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        # Only run for the Python version specified in the git tag.
+        #
+        # This if could be moved to the parent `job` section when it's
+        # supported by GitHub (https://github.com/community/community/discussions/37883)
+        if: startsWith(github.event.ref,  format('refs/tags/python/{0}+', matrix.version))
+        uses: actions/checkout@v3
+      - name: Build Python
+        # Only run for the Python version specified in the git tag.
+        #
+        # This if could be moved to the parent `job` section when it's
+        # supported by GitHub (https://github.com/community/community/discussions/37883)
+        if: startsWith(github.event.ref, format('refs/tags/python/{0}+', matrix.version))
+        run: make python/v${{ matrix.version }}
+      - name: Rename release artifacts
+        # Only run for the Python version specified in the git tag.
+        #
+        # This if could be moved to the parent `job` section when it's
+        # supported by GitHub (https://github.com/community/community/discussions/37883)
+        if: startsWith(github.event.ref, format('refs/tags/python/{0}+', matrix.version))
+        shell: bash
+        run: |
+          sudo mv python/build-output/python/v${{ matrix.version }}/bin/python{,-${{ matrix.version }}}.wasm
+      - name: Bundle Python with standard libraries
+        # Only run for the Python version specified in the git tag.
+        #
+        # This if could be moved to the parent `job` section when it's
+        # supported by GitHub (https://github.com/community/community/discussions/37883)
+        if: startsWith(github.event.ref, format('refs/tags/python/{0}+', matrix.version))
+        run: |
+          pushd python/build-output/python/v${{ matrix.version }}
+          sudo zip -qq -r python-aio-${{ matrix.version }}.zip bin/python-${{ matrix.version }}.wasm usr
+          popd
+      - name: Create release
+        # Only run for the Python version specified in the git tag.
+        #
+        # This if could be moved to the parent `job` section when it's
+        # supported by GitHub (https://github.com/community/community/discussions/37883)
+        if: startsWith(github.event.ref, format('refs/tags/python/{0}+', matrix.version))
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh release create --generate-notes ${{ github.ref_name }} || true
+      - name: Append Python release assets
+        # Only run for the Python version specified in the git tag.
+        #
+        # This if could be moved to the parent `job` section when it's
+        # supported by GitHub (https://github.com/community/community/discussions/37883)
+        if: ${{ startsWith(github.event.ref, format('refs/tags/python/{0}+', matrix.version))}}
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh release upload ${{ github.ref_name }} \
+            python/build-output/python/v${{ matrix.version }}/python-aio-${{ matrix.version }}.zip
+      - name: Generate release assets digests
+        # Only run for the Python version specified in the git tag.
+        #
+        # This if could be moved to the parent `job` section when it's
+        # supported by GitHub (https://github.com/community/community/discussions/37883)
+        if: ${{ startsWith(github.event.ref, format('refs/tags/python/{0}+', matrix.version))}}
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          for asset in python/build-output/python/v${{ matrix.version }}/python-aio-${{ matrix.version }}.zip; do
+            sha256sum "$asset" | sudo tee "$asset.sha256sum" > /dev/null
+          done
+      - name: Append release assets digests
+        # Only run for the Python version specified in the git tag.
+        #
+        # This if could be moved to the parent `job` section when it's
+        # supported by GitHub (https://github.com/community/community/discussions/37883)
+        if: ${{ startsWith(github.event.ref, format('refs/tags/python/{0}+', matrix.version))}}
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh release upload ${{ github.ref_name }} \
+            python/build-output/python/v${{ matrix.version }}/python-aio-${{ matrix.version }}.zip.sha256sum
+

Makefile

@@ -12,6 +12,10 @@ php/wasmedge-php-7.4.32:
 ruby/v*:
 	make -C ruby $(subst ruby/,,$@)
 
+.PHONY: python/v*
+python/v*:
+	make -C python $(subst python/,,$@)
+
 .PHONY: clean
 clean:
 	make -C php clean

Makefile.builders

@@ -8,4 +8,8 @@ wasm-base:
 
 .PHONY: wasi-builder-19
 wasi-builder-19: wasm-base
-	docker build --build-arg WASM_BASE=$(WASM_BASE_TAG) --build-arg WASI_SDK_VERSION=19 -f ${BUILDER_ROOT_DIR}/Dockerfile.wasi-builder -t ghcr.io/vmware-labs/wasi-builder:19 ${BUILDER_ROOT_DIR}
+	docker build --build-arg WASM_BASE=$(WASM_BASE_TAG) --build-arg WASI_SDK_VERSION=19 -f ${BUILDER_ROOT_DIR}/Dockerfile.wasi-builder -t ghcr.io/vmware-labs/wasi-builder:19 ${BUILDER_ROOT_DIR}
+
+.PHONY: wasi-builder-16
+wasi-builder-16: wasm-base
+	docker build --build-arg WASM_BASE=$(WASM_BASE_TAG) --build-arg WASI_SDK_VERSION=16 -f ${BUILDER_ROOT_DIR}/Dockerfile.wasi-builder -t ghcr.io/vmware-labs/wasi-builder:16 ${BUILDER_ROOT_DIR}

libs/sqlite/version-3.39.2/wl-build.sh

@@ -7,7 +7,7 @@ then
 fi
 
 # export CFLAGS_CONFIG="-O3 -g"
-export CFLAGS_CONFIG="-O2"
+export CFLAGS_CONFIG="-O0"
 
 export CFLAGS_WASI="--sysroot=${WASI_SYSROOT} -I./wasmlabs-stubs -D_WASI_EMULATED_MMAN -D_WASI_EMULATED_GETPID -D_WASI_EMULATED_SIGNAL -D_WASI_EMULATED_PROCESS_CLOCKS"
 export LDFLAGS_WASI="--sysroot=${WASI_SYSROOT} -lwasi-emulated-mman -lwasi-emulated-getpid -lwasi-emulated-signal -lwasi-emulated-process-clocks"

python/.dockerignore

@@ -0,0 +1,2 @@
+build-output
+build-staging

python/Dockerfile

@@ -0,0 +1,12 @@
+ARG WASI_SDK_VERSION=16
+FROM ghcr.io/vmware-labs/wasi-builder:${WASI_SDK_VERSION}
+
+# If more capabilities are required from the build-erpython, consult this
+# github workflow configuration for a list of possible dependencies -
+# https://github.com/python/cpython/blob/main/.github/workflows/posix-deps-apt.sh
+RUN DEBIAN_FRONTEND=noninteractive apt install -y \
+      tcl \
+      uuid-dev \
+      zlib1g-dev
+
+ADD . /wlr/php

python/Makefile

@@ -0,0 +1,19 @@
+WASI_SDK_VERSION ?= 16
+# NOTE - the default python build is failing with wasi-sdk-19
+
+ROOT_DIR := $(dir $(realpath $(lastword $(MAKEFILE_LIST))))
+
+include ../Makefile.builders
+
+.PHONY: python-builder
+python-builder: wasi-builder-16
+	docker build -f ${ROOT_DIR}/Dockerfile --build-arg WASI_SDK_VERSION=$(WASI_SDK_VERSION) -t ghcr.io/vmware-labs/python-builder:wasi-$(WASI_SDK_VERSION) ${ROOT_DIR}
+
+.PHONY: v*
+v*: python-builder
+	mkdir -p build-output build-staging
+	docker run --rm -e WASMLABS_RUNTIME -v ${ROOT_DIR}/build-output:/wlr/build-output -v ${ROOT_DIR}/build-staging:/wlr/build-staging ghcr.io/vmware-labs/python-builder:wasi-${WASI_SDK_VERSION} ./wl-make.sh python/$@
+
+.PHONY: clean
+clean:
+	rm -rf build-output build-staging

python/v3.11.1/wl-build-deps.sh

@@ -25,3 +25,10 @@ $WASMLABS_MAKE ${WASMLABS_REPO_ROOT}/libs/zlib/v1.2.11 || exit 1
 
 export CFLAGS_DEPENDENCIES="-I${WASMLABS_OUTPUT_BASE}/zlib/v1.2.11/include ${CFLAGS_DEPENDENCIES}"
 export LDFLAGS_DEPENDENCIES="-L${WASMLABS_OUTPUT_BASE}/zlib/v1.2.11/lib ${LDFLAGS_DEPENDENCIES}"
+
+
+### sqlite3
+$WASMLABS_MAKE ${WASMLABS_REPO_ROOT}/libs/sqlite/version-3.39.2 || exit 1
+
+export CFLAGS_DEPENDENCIES="-I${WASMLABS_OUTPUT_BASE}/sqlite/version-3.39.2/include ${CFLAGS_DEPENDENCIES}"
+export LDFLAGS_DEPENDENCIES="-L${WASMLABS_OUTPUT_BASE}/sqlite/version-3.39.2/lib ${LDFLAGS_DEPENDENCIES}"

python/v3.11.1/wl-build.sh

@@ -26,8 +26,7 @@ else
     make distclean || exit 1
 fi
 
-# export CFLAGS_CONFIG="-O3 -g"
-export CFLAGS_CONFIG="-O3"
+export CFLAGS_CONFIG="-O0"
 
 export CFLAGS="${CFLAGS_CONFIG} ${CFLAGS_DEPENDENCIES} ${CFLAGS}"
 export LDFLAGS="${LDFLAGS_DEPENDENCIES} ${LDFLAGS}"
@@ -39,6 +38,15 @@ then
     export PYTHON_WASM_CONFIGURE=" --with-wasm-runtime=${WASMLABS_RUNTIME} ${PYTHON_WASM_CONFIGURE}"
 fi
 
+# By exporting WASMLABS_SKIP_WASM_OPT envvar during the build, the
+# wasm-opt wrapper in the wasm-base image will be a dummy wrapper that
+# is effectively a NOP.
+#
+# This is due to https://github.com/llvm/llvm-project/issues/55781, so
+# that we get to choose which optimization passes are executed after
+# the artifacts have been built.
+export WASMLABS_SKIP_WASM_OPT=1
+
 logStatus "Configuring build with '${PYTHON_WASM_CONFIGURE}'... "
 CONFIG_SITE=./Tools/wasm/config.site-wasm32-wasi ./configure -C --host=wasm32-wasi --build=$(./config.guess) ${PYTHON_WASM_CONFIGURE} || exit 1
 
@@ -47,11 +55,16 @@ export MAKE_TARGETS='python.wasm wasm_stdlib'
 logStatus "Building '${MAKE_TARGETS}'... "
 make -j ${MAKE_TARGETS} || exit 1
 
+unset WASMLABS_SKIP_WASM_OPT
+
 logStatus "Preparing artifacts... "
 mkdir -p ${WASMLABS_OUTPUT}/bin 2>/dev/null || exit 1
 mkdir -p ${WASMLABS_OUTPUT}/usr/local/lib 2>/dev/null || exit 1
 
-cp python.wasm ${WASMLABS_OUTPUT}/bin/python${WASMLABS_RUNTIME:+-$WASMLABS_RUNTIME}.wasm || exit 1
-cp usr/local/lib/python3.11.zip ${WASMLABS_OUTPUT}/usr/local/lib/ || exit 1
+logStatus "Optimizing python binary..."
+wasm-opt -O2 -o ${WASMLABS_OUTPUT}/bin/python${WASMLABS_RUNTIME:+-$WASMLABS_RUNTIME}.wasm python.wasm || exit 1
+
+logStatus "Copying standard libs..."
+cp usr/local/lib/python311.zip ${WASMLABS_OUTPUT}/usr/local/lib/ || exit 1
 
 logStatus "DONE. Artifacts in ${WASMLABS_OUTPUT}"