diff --git a/.github/workflows/build-python.yaml b/.github/workflows/build-python.yaml index db823f02..f36b3b2a 100644 --- a/.github/workflows/build-python.yaml +++ b/.github/workflows/build-python.yaml @@ -13,7 +13,7 @@ jobs: fail-fast: false matrix: flavor: ["", "-wasmedge", "-aio", "-aio-wasmedge"] - version: ["3.11.3", "3.11.4"] + version: ["3.11.3", "3.11.4", "3.12.0"] runs-on: ubuntu-latest steps: - name: Checkout repository diff --git a/.gitignore b/.gitignore index 607354b8..b1d256d4 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ build-staging node_modules target .vscode/** +.python-version *.log # Except this file !.gitignore diff --git a/Makefile b/Makefile index c566abc0..e515d2d9 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,7 @@ python/v*: $(eval $(call create_flavor_targets,python,v3.11.1,aio wasmedge aio-wasmedge)) $(eval $(call create_flavor_targets,python,v3.11.3,aio wasmedge aio-wasmedge)) $(eval $(call create_flavor_targets,python,v3.11.4,aio wasmedge aio-wasmedge)) +$(eval $(call create_flavor_targets,python,v3.12.0,aio wasmedge aio-wasmedge)) .PHONY: oci-python-3.11.1 oci-python-3.11.1: python/v3.11.1 diff --git a/python/Dockerfile b/python/Dockerfile index 54d5409e..045488a0 100644 --- a/python/Dockerfile +++ b/python/Dockerfile @@ -1,26 +1,26 @@ ARG WASI_SDK_VERSION -ARG PY_VERSION=3.11.0 -ARG PY_SHORT_VERSION=3.11 - -FROM ghcr.io/vmware-labs/wasmlabs/wasi-builder:${WASI_SDK_VERSION} as build-host-python -ARG PY_VERSION -ARG PY_SHORT_VERSION -RUN DEBIAN_FRONTEND=noninteractive apt install -y \ - zlib1g-dev -WORKDIR /tmp -RUN wget -c https://www.python.org/ftp/python/${PY_VERSION}/Python-${PY_VERSION}.tar.xz && tar -Jxf Python-${PY_VERSION}.tar.xz -WORKDIR /tmp/Python-${PY_VERSION} -RUN ./configure --enable-optimizations --prefix=/opt/python${PY_SHORT_VERSION} -RUN make -j4 && make install +ARG PY_VERSIONS="3.12.0 3.11.4" +ARG PY_DEFAULT_VERSION=3.12.0 FROM ghcr.io/vmware-labs/wasmlabs/wasi-builder:${WASI_SDK_VERSION} -ARG PY_SHORT_VERSION +ARG PY_VERSIONS +ARG PY_DEFAULT_VERSION # If more capabilities are required from the python-builder, consult this # github workflow configuration for a list of possible dependencies - # https://github.com/python/cpython/blob/main/.github/workflows/posix-deps-apt.sh -RUN DEBIAN_FRONTEND=noninteractive apt install -y \ - tcl +RUN DEBIAN_FRONTEND=noninteractive apt update && \ + apt install -y --no-install-recommends \ + libssl-dev \ + tcl \ + zlib1g-dev + +ENV HOME="/root" +WORKDIR ${HOME} +RUN git clone --depth=1 https://github.com/pyenv/pyenv.git .pyenv +ENV PYENV_ROOT="${HOME}/.pyenv" +ENV PATH="${PYENV_ROOT}/shims:${PYENV_ROOT}/bin:${PATH}" + +RUN pyenv install ${PY_VERSIONS} +RUN pyenv global ${PY_DEFAULT_VERSION} -COPY --from=build-host-python /opt/python${PY_SHORT_VERSION}/ /opt/python${PY_SHORT_VERSION}/ -RUN update-alternatives --install /usr/bin/python3 python3 /opt/python${PY_SHORT_VERSION}/bin/python${PY_SHORT_VERSION} 110 -RUN update-alternatives --install /usr/bin/python${PY_SHORT_VERSION} python${PY_SHORT_VERSION} /opt/python${PY_SHORT_VERSION}/bin/python${PY_SHORT_VERSION} 110 +WORKDIR /wlr diff --git a/python/test/run_me.sh b/python/test/run_me.sh index 6e3591ba..36ca3584 100755 --- a/python/test/run_me.sh +++ b/python/test/run_me.sh @@ -10,7 +10,7 @@ TESTS=$(for t in */test.py; do echo $(dirname $t); done) status=0 -for test_dir in "$TESTS"; do +for test_dir in ${TESTS}; do echo "Running test in '${test_dir}'..." if diff ${test_dir}/test.stdout <($WLR_TEST_RUNTIME \ --mapdir /usr::${WLR_OUTPUT}/usr \ diff --git a/python/v3.11.1/wlr-build.sh b/python/v3.11.1/wlr-build.sh index c6c49679..60a55796 100644 --- a/python/v3.11.1/wlr-build.sh +++ b/python/v3.11.1/wlr-build.sh @@ -6,6 +6,8 @@ then exit 1 fi +pyenv local ${WLR_PY_BUILDER_VERSION} + cd "${WLR_SOURCE_PATH}" if [[ "${WLR_BUILD_FLAVOR}" == *"aio"* ]] diff --git a/python/v3.11.1/wlr-env-repo.sh b/python/v3.11.1/wlr-env-repo.sh index 9541873d..34180c15 100644 --- a/python/v3.11.1/wlr-env-repo.sh +++ b/python/v3.11.1/wlr-env-repo.sh @@ -7,6 +7,7 @@ then unset WLR_ENV_NAME unset WLR_PACKAGE_VERSION unset WLR_PACKAGE_NAME + unset WLR_PY_BUILDER_VERSION return fi @@ -15,3 +16,4 @@ export WLR_REPO_BRANCH=v3.11.1 export WLR_ENV_NAME=python/v3.11.1 export WLR_PACKAGE_VERSION=3.11.1 export WLR_PACKAGE_NAME=python +export WLR_PY_BUILDER_VERSION=3.11.4 diff --git a/python/v3.11.3/wlr-build.sh b/python/v3.11.3/wlr-build.sh index 430403b3..fdcafac5 100644 --- a/python/v3.11.3/wlr-build.sh +++ b/python/v3.11.3/wlr-build.sh @@ -6,6 +6,8 @@ then exit 1 fi +pyenv local ${WLR_PY_BUILDER_VERSION} + cd "${WLR_SOURCE_PATH}" if [[ "${WLR_BUILD_FLAVOR}" == *"aio"* ]] diff --git a/python/v3.11.3/wlr-env-repo.sh b/python/v3.11.3/wlr-env-repo.sh index 28469606..e3cae212 100644 --- a/python/v3.11.3/wlr-env-repo.sh +++ b/python/v3.11.3/wlr-env-repo.sh @@ -7,6 +7,7 @@ then unset WLR_ENV_NAME unset WLR_PACKAGE_VERSION unset WLR_PACKAGE_NAME + unset WLR_PY_BUILDER_VERSION return fi @@ -15,3 +16,4 @@ export WLR_REPO_BRANCH=v3.11.3 export WLR_ENV_NAME=python/v3.11.3 export WLR_PACKAGE_VERSION=3.11.3 export WLR_PACKAGE_NAME=python +export WLR_PY_BUILDER_VERSION=3.11.4 diff --git a/python/v3.11.4/wlr-build.sh b/python/v3.11.4/wlr-build.sh index 430403b3..fdcafac5 100644 --- a/python/v3.11.4/wlr-build.sh +++ b/python/v3.11.4/wlr-build.sh @@ -6,6 +6,8 @@ then exit 1 fi +pyenv local ${WLR_PY_BUILDER_VERSION} + cd "${WLR_SOURCE_PATH}" if [[ "${WLR_BUILD_FLAVOR}" == *"aio"* ]] diff --git a/python/v3.11.4/wlr-env-repo.sh b/python/v3.11.4/wlr-env-repo.sh index 4d4946b1..7c36cccc 100644 --- a/python/v3.11.4/wlr-env-repo.sh +++ b/python/v3.11.4/wlr-env-repo.sh @@ -7,6 +7,7 @@ then unset WLR_ENV_NAME unset WLR_PACKAGE_VERSION unset WLR_PACKAGE_NAME + unset WLR_PY_BUILDER_VERSION return fi @@ -15,3 +16,4 @@ export WLR_REPO_BRANCH=v3.11.4 export WLR_ENV_NAME=python/v3.11.4 export WLR_PACKAGE_VERSION=3.11.4 export WLR_PACKAGE_NAME=python +export WLR_PY_BUILDER_VERSION=3.11.4 diff --git a/python/v3.12.0/wlr-build.sh b/python/v3.12.0/wlr-build.sh new file mode 100644 index 00000000..fdca5102 --- /dev/null +++ b/python/v3.12.0/wlr-build.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash + +if [[ ! -v WLR_ENV ]] +then + echo "WLR build environment is not set" + exit 1 +fi + +pyenv local ${WLR_PY_BUILDER_VERSION} + +cd "${WLR_SOURCE_PATH}" + +if [[ "${WLR_BUILD_FLAVOR}" == *"aio"* ]] +then + source ${WLR_REPO_ROOT}/scripts/build-helpers/wlr_wasi_vfs.sh + export LDFLAGS="$(wlr_wasi_vfs_get_link_flags) ${LDFLAGS}" +fi + +source ${WLR_REPO_ROOT}/scripts/build-helpers/wlr_pkg_config.sh + +export CFLAGS_CONFIG="-O0" + + +# This fails with upgraded clang for wasi-sdk19 and later. Disabled on cpython main. +# +# PyModule_AddIntMacro(module, CLOCK_MONOTONIC) and the like cause this. +# In all POSIX variants CLOCK_MONOTONIC is a numeric constant, so python imports it as int macro +# However, in wasi-libc clockid_t is defined as a pointer to struct __clockid. + +export CFLAGS_CONFIG="${CFLAGS_CONFIG} -Wno-int-conversion" + +export CFLAGS="${CFLAGS_CONFIG} ${CFLAGS_DEPENDENCIES} ${CFLAGS}" +export LDFLAGS="${LDFLAGS_DEPENDENCIES} ${LDFLAGS}" + +export PYTHON_WASM_CONFIGURE="--with-build-python=python3" + +if [[ "${WLR_BUILD_FLAVOR}" == *"wasmedge"* ]] +then + if [[ ! -v WABT_ROOT ]] + then + echo "WABT_ROOT is needed to patch imports for wasmedge" + exit 1 + fi +fi + +# By exporting WLR_SKIP_WASM_OPT envvar during the build, the +# wasm-opt wrapper in the wasm-base image will be a dummy wrapper that +# is effectively a NOP. +# +# This is due to https://github.com/llvm/llvm-project/issues/55781, so +# that we get to choose which optimization passes are executed after +# the artifacts have been built. +export WLR_SKIP_WASM_OPT=1 + +if [[ -z "$WLR_SKIP_CONFIGURE" ]]; then + logStatus "Configuring build with '${PYTHON_WASM_CONFIGURE}'... " + CONFIG_SITE=./Tools/wasm/config.site-wasm32-wasi ./configure -C --host=wasm32-wasi --build=$(./config.guess) ${PYTHON_WASM_CONFIGURE} || exit 1 +else + logStatus "Skipping configure..." +fi + +export MAKE_TARGETS='python.wasm wasm_stdlib' + +logStatus "Building '${MAKE_TARGETS}'... " +make -j ${MAKE_TARGETS} || exit 1 + +unset WLR_SKIP_WASM_OPT + +if [[ "${WLR_BUILD_FLAVOR}" == *"aio"* ]] +then + logStatus "Packing with wasi-vfs" + wlr_wasi_vfs_cli pack python.wasm --mapdir /usr::$PWD/usr -o python.wasm || exit 1 +fi + +logStatus "Optimizing python binary..." +wasm-opt -O2 -o python-optimized.wasm python.wasm || exit 1 + +if [[ "${WLR_BUILD_FLAVOR}" == *"wasmedge"* ]] +then + logStatus "Patching python binary for wasmedge..." + ${WLR_REPO_ROOT}/scripts/build-helpers/patch_wasmedge_wat_sock_accept.sh python-optimized.wasm || exit 1 +fi + +logStatus "Preparing artifacts... " +TARGET_PYTHON_BINARY=bin/python-${WLR_PACKAGE_VERSION}.wasm + +mkdir -p ${WLR_OUTPUT}/bin 2>/dev/null || exit 1 + +if [[ "${WLR_BUILD_FLAVOR}" == *"aio"* ]] +then + cp -v python-optimized.wasm ${WLR_OUTPUT}/${TARGET_PYTHON_BINARY} || exit 1 +else + mkdir -p ${WLR_OUTPUT}/usr 2>/dev/null || exit 1 + cp -v python-optimized.wasm ${WLR_OUTPUT}/${TARGET_PYTHON_BINARY} || exit 1 + cp -TRv usr ${WLR_OUTPUT}/usr || exit 1 +fi + +if [[ "${WLR_BUILD_FLAVOR}" != *"aio"* && "${WLR_BUILD_FLAVOR}" != *"wasmedge"* ]] +then + + logStatus "Install includes..." + make inclinstall \ + prefix=${WLR_OUTPUT} \ + libdir=${WLR_OUTPUT}/lib/wasm32-wasi \ + pkgconfigdir=${WLR_OUTPUT}/lib/wasm32-wasi/pkgconfig || exit 1 + + logStatus "Create libpython3.12-aio.a" +(${AR} -M </dev/null || exit 1 + cp -v libpython3.12-aio.a ${WLR_OUTPUT}/lib/wasm32-wasi/libpython3.12.a || exit 1 + + logStatus "Generating pkg-config file for libpython3.12.a" + DESCRIPTION="libpython3.12 allows embedding the CPython interpreter" + EXTRA_LINK_FLAGS="-lpython3.12 -Wl,-z,stack-size=524288 -Wl,--stack-first -Wl,--initial-memory=10485760 -lwasi-emulated-getpid -lwasi-emulated-signal -lwasi-emulated-process-clocks" + + PC_INCLUDE_SUBDIR=python3.12 wlr_pkg_config_create_pc_file "libpython3.12" "${WLR_PACKAGE_VERSION}" "${DESCRIPTION}" "${EXTRA_LINK_FLAGS}" || exit 1 + + WLR_PACKAGE_EXTRA_DIRS=usr wlr_package_lib || exit 1 + WLR_PACKAGE_LIST="${TARGET_PYTHON_BINARY} usr" wlr_package || exit 1 + +elif [[ "${WLR_BUILD_FLAVOR}" == *"aio"* ]]; then + # skip 'aio' in the name + FLAVOR_SUFFIX=$(echo ${WLR_BUILD_FLAVOR} | sed 's/-\?aio//g') + PUBLISHED_PYTHON_BINARY=python-${WLR_PACKAGE_VERSION}${FLAVOR_SUFFIX}.wasm + cp -v ${WLR_OUTPUT}/${TARGET_PYTHON_BINARY} ${WLR_OUTPUT_BASE}/${PUBLISHED_PYTHON_BINARY} || exit 1 + +else + WLR_PACKAGE_LIST="${TARGET_PYTHON_BINARY} usr" wlr_package || exit 1 +fi + +logStatus "DONE. Artifacts in ${WLR_OUTPUT}" diff --git a/python/v3.12.0/wlr-env-repo.sh b/python/v3.12.0/wlr-env-repo.sh new file mode 100644 index 00000000..96f070fb --- /dev/null +++ b/python/v3.12.0/wlr-env-repo.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +if [[ $1 == "--unset" ]] +then + unset WLR_REPO + unset WLR_REPO_BRANCH + unset WLR_ENV_NAME + unset WLR_PACKAGE_VERSION + unset WLR_PACKAGE_NAME + unset WLR_PY_BUILDER_VERSION + return +fi + +export WLR_REPO=https://github.com/python/cpython +export WLR_REPO_BRANCH=v3.12.0 +export WLR_ENV_NAME=python/v3.12.0 +export WLR_PACKAGE_VERSION=3.12.0 +export WLR_PACKAGE_NAME=python +export WLR_PY_BUILDER_VERSION=3.12.0 diff --git a/python/v3.12.0/wlr-info.json b/python/v3.12.0/wlr-info.json new file mode 100644 index 00000000..ea19e62e --- /dev/null +++ b/python/v3.12.0/wlr-info.json @@ -0,0 +1,24 @@ +{ + "deps": { + "libuuid": { + "build_target": "libs/libuuid/v1.0.3", + "required_file": "lib/wasm32-wasi/libuuid.a", + "url": "https://github.com/vmware-labs/webassembly-language-runtimes/releases/download/libs%2Flibuuid%2F1.0.3%2B20230623-2993864/libuuid-1.0.3-wasi-sdk-20.0.tar.gz" + }, + "zlib": { + "build_target": "libs/zlib/v1.2.13", + "required_file": "lib/wasm32-wasi/libz.a", + "url": "https://github.com/vmware-labs/webassembly-language-runtimes/releases/download/libs%2Fzlib%2F1.2.13%2B20230623-2993864/libz-1.2.13-wasi-sdk-20.0.tar.gz" + }, + "SQLite": { + "build_target": "libs/sqlite/v3.42.0", + "required_file": "lib/wasm32-wasi/libsqlite3.a", + "url": "https://github.com/vmware-labs/webassembly-language-runtimes/releases/download/libs%2Fsqlite%2F3.42.0%2B20230623-2993864/libsqlite-3.42.0-wasi-sdk-20.0.tar.gz" + }, + "bzip2": { + "build_target": "libs/bzip2/v1.0.8", + "required_file": "lib/wasm32-wasi/libbz2.a", + "url": "https://github.com/vmware-labs/webassembly-language-runtimes/releases/download/libs%2Fbzip2%2F1.0.8%2B20230623-2993864/libbzip2-1.0.8-wasi-sdk-20.0.tar.gz" + } + } +} \ No newline at end of file diff --git a/python/v3.12.0/wlr-tag.sh b/python/v3.12.0/wlr-tag.sh new file mode 100644 index 00000000..39e6db73 --- /dev/null +++ b/python/v3.12.0/wlr-tag.sh @@ -0,0 +1 @@ +export WLR_TAG="python/3.12.0" diff --git a/python/v3.12.0/wlr-test.sh b/python/v3.12.0/wlr-test.sh new file mode 100644 index 00000000..072e8c32 --- /dev/null +++ b/python/v3.12.0/wlr-test.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +if [[ ! -v WLR_ENV ]] +then + echo "WLR build environment is not set" + exit 1 +fi + +if [[ "${WLR_BUILD_FLAVOR}" == *"wasmedge"* ]] +then + WLR_BINARY_WUFFIX=-wasmedge +fi + +export WLR_TESTED_MODULE="${WLR_OUTPUT}/bin/python-${WLR_PACKAGE_VERSION}${WLR_BINARY_WUFFIX}.wasm" +export TEST_ROOT=${WLR_ENV}/../test + +(cd ${TEST_ROOT}; ./run_me.sh)