Add WASM Python host

trinodb · Dec 8, 2024 · a72158b · a72158b
1 parent 093117a
commit a72158b
Show file tree

Hide file tree

Showing 10 changed files with 1,456 additions and 1 deletion.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -6,4 +6,11 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - run: true
+      - uses: actions/checkout@v4
+      - uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: 21
+      - run: ./build-image.sh
+      - run: mvn dependency:go-offline -B -V
+      - run: mvn install -B
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -0,0 +1,15 @@
+cmake_minimum_required(VERSION 3.28)
+
+project(python-host)
+
+set(CMAKE_EXECUTABLE_SUFFIX ".wasm")
+
+add_compile_options(-Werror -Wall -Wextra -Wimplicit-fallthrough)
+
+add_link_options(-Wl,--max-memory=33554432)
+
+add_executable(python-host pyhost.c)
+
+find_package(Python COMPONENTS Development)
+
+target_link_libraries(python-host PUBLIC Python::Python wasi_vfs)
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,106 @@
+FROM ubuntu
+
+RUN <<EOF
+apt-get update
+apt-get install -y curl build-essential python3 git cmake pkg-config busybox
+EOF
+
+ARG TARGETARCH
+
+ENV WASI_SDK_VERSION=24
+ENV WASI_SDK_PATH=/opt/wasi-sdk
+ENV WASI_SDK_SYSROOT=${WASI_SDK_PATH}/share/wasi-sysroot
+ENV WASI_SDK_LIBDIR=${WASI_SDK_SYSROOT}/lib/wasm32-wasi
+ENV WASMTIME_VERSION=26.0.0
+ENV WIZER_VERSION=7.0.5
+ENV WASI_VFS_VERSION=0.5.4
+ENV PYTHON_PATH=/opt/wasi-python
+
+RUN <<EOF
+mkdir ${WASI_SDK_PATH}
+WASI_SDK_ARCH=$(echo ${TARGETARCH} | sed 's/amd64/x86_64/')
+curl -L https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-${WASI_SDK_ARCH}-linux.tar.gz | \
+    tar -xz --strip-components 1 -C ${WASI_SDK_PATH}
+EOF
+
+RUN <<EOF
+WASMTIME_ARCH=$(echo ${TARGETARCH} | sed -e 's/amd64/x86_64/' -e 's/arm64/aarch64/')
+WASMTIME_NAME=wasmtime-v${WASMTIME_VERSION}-${WASMTIME_ARCH}-linux
+curl -L https://github.com/bytecodealliance/wasmtime/releases/download/v${WASMTIME_VERSION}/${WASMTIME_NAME}.tar.xz | \
+    tar -xJ --strip-components 1 -C /usr/local/bin ${WASMTIME_NAME}/wasmtime
+EOF
+
+RUN <<EOF
+WIZER_ARCH=$(echo ${TARGETARCH} | sed -e 's/amd64/x86_64/' -e 's/arm64/aarch64/')
+WIZER_NAME=wizer-v${WIZER_VERSION}-${WIZER_ARCH}-linux
+curl -L https://github.com/bytecodealliance/wizer/releases/download/v${WIZER_VERSION}/${WIZER_NAME}.tar.xz | \
+    tar -xJ --strip-components 1 -C /usr/local/bin ${WIZER_NAME}/wizer
+EOF
+
+RUN <<EOF
+WASI_VFS_ARCH=$(echo ${TARGETARCH} | sed -e 's/amd64/x86_64/' -e 's/arm64/aarch64/')
+curl -L https://github.com/kateinoigakukun/wasi-vfs/releases/download/v${WASI_VFS_VERSION}/wasi-vfs-cli-${WASI_VFS_ARCH}-unknown-linux-gnu.zip | \
+    busybox unzip -d /usr/local/bin -
+chmod +x /usr/local/bin/wasi-vfs
+curl -L https://github.com/kateinoigakukun/wasi-vfs/releases/download/v${WASI_VFS_VERSION}/libwasi_vfs-wasm32-unknown-unknown.zip | \
+    busybox unzip -d ${WASI_SDK_LIBDIR} -
+EOF
+
+RUN <<EOF
+mkdir -p /build/zlib
+curl -L https://www.zlib.net/zlib-1.3.1.tar.gz | \
+  tar -xz --strip-components 1 -C /build/zlib
+EOF
+
+RUN <<EOF
+cd /build/zlib
+prefix=${WASI_SDK_SYSROOT} \
+libdir=${WASI_SDK_LIBDIR} \
+pkgconfigdir=${WASI_SDK_SYSROOT}/lib/pkgconfig \
+CC="${WASI_SDK_PATH}/bin/clang --sysroot=${WASI_SDK_SYSROOT}" \
+AR="${WASI_SDK_PATH}/bin/ar" \
+./configure
+make install
+EOF
+
+RUN <<EOF
+mkdir -p /build/cpython
+curl -L https://www.python.org/ftp/python/3.13.0/Python-3.13.0.tgz | \
+    tar -xz --strip-components 1 -C /build/cpython
+EOF
+
+RUN <<EOF
+cd /build/cpython
+python3 Tools/wasm/wasi.py configure-build-python
+python3 Tools/wasm/wasi.py make-build-python
+EOF
+
+RUN <<EOF
+cd /build/cpython
+python3 Tools/wasm/wasi.py configure-host -- \
+    CFLAGS='-Os' --prefix=${PYTHON_PATH} --disable-test-modules
+python3 Tools/wasm/wasi.py make-host
+make -C cross-build/wasm32-wasi install COMPILEALL_OPTS='-j0 -b'
+EOF
+
+RUN <<EOF
+cd /build/cpython/cross-build/wasm32-wasi
+${WASI_SDK_PATH}/bin/ar -M <<AR
+create ${PYTHON_PATH}/lib/libpython3.13.a
+addlib libpython3.13.a
+addlib Modules/expat/libexpat.a
+addlib Modules/_decimal/libmpdec/libmpdec.a
+addlib Modules/_hacl/libHacl_Hash_SHA2.a
+addlib ${WASI_SDK_LIBDIR}/libz.a
+save
+end
+AR
+EOF
+
+RUN <<EOF
+cd ${PYTHON_PATH}/lib/python3.13
+find . -name __pycache__ -exec rm -rf {} \;
+rm -rf config-3.13-wasm32-wasi
+rm -rf _*_support* _pyrepl bdb ensurepip doctest* idlelib pdb pydoc*
+rm -rf sqlite3 ssl* subprocess* tkinter turtle* venv webbrowser*
+EOF
diff --git a/build-image.sh b/build-image.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -eu
+cd "${BASH_SOURCE%/*}"
+
+BUILD_IMAGE=${BUILD_IMAGE:-trinodb/wasm-python}
+
+docker build . --load --tag $BUILD_IMAGE
diff --git a/build-wasm.sh b/build-wasm.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+BUILD_TYPE=$([[ "${1:-}" == "--debug" ]] && echo "Debug" || echo "Release")
+
+WASI_SDK_PATH="${WASI_SDK_PATH:-/opt/wasi-sdk}"
+WIZER="${WIZER:-/usr/local/bin/wizer}"
+PYTHON_PATH="${PYTHON_PATH:-/opt/wasi-python}"
+
+TARGET_DIR=target/wasm
+
+export CMAKE_EXTRA_ARGS="
+    -DCMAKE_BUILD_TYPE=${BUILD_TYPE}
+    -DWASI_SDK_PREFIX=${WASI_SDK_PATH}
+    -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK_PATH}/share/cmake/wasi-sdk.cmake
+    -DCMAKE_PREFIX_PATH=/opt/wasi-python"
+
+cmake -B ${TARGET_DIR} ${CMAKE_EXTRA_ARGS} .
+cmake --build ${TARGET_DIR} --verbose
+
+rm -rf "${TARGET_DIR}"/python
+
+cp -a ${PYTHON_PATH}/lib/python3.13 ${TARGET_DIR}/python
+cp /work/trino.py ${TARGET_DIR}/python/site-packages/
+
+wasi-vfs pack ${TARGET_DIR}/python-host.wasm \
+    --dir ${TARGET_DIR}/python::/opt/wasi-python/lib/python3.13 \
+    --output ${TARGET_DIR}/python-host-packed.wasm
+
+rm -rf "${TARGET_DIR}"/empty
+mkdir ${TARGET_DIR}/empty
+
+env - TERM=dumb PYTHONDONTWRITEBYTECODE=1 ${WIZER} \
+    --wasm-bulk-memory true \
+    --allow-wasi \
+    --inherit-stdio true \
+    --inherit-env true \
+    --init-func _start \
+    --keep-init-func false \
+    --mapdir /guest::${TARGET_DIR}/empty \
+    -o ${TARGET_DIR}/python-host-opt.wasm \
+    ${TARGET_DIR}/python-host-packed.wasm
+
+rm -rf "${TARGET_DIR}"/python/site-packages/__pycache__
diff --git a/build.sh b/build.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -eu
+cd "${BASH_SOURCE%/*}"
+
+BUILD_IMAGE=${BUILD_IMAGE:-trinodb/wasm-python}
+
+docker run -t -v"$PWD":/work -w /work $BUILD_IMAGE ./build-wasm.sh "$@"
diff --git a/pom.xml b/pom.xml
@@ -0,0 +1,136 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" child.project.url.inherit.append.path="false" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>io.airlift</groupId>
+        <artifactId>airbase</artifactId>
+        <version>202</version>
+    </parent>
+
+    <groupId>io.trino</groupId>
+    <artifactId>trino-wasm-python</artifactId>
+    <version>3.13-1-SNAPSHOT</version>
+    <packaging>jar</packaging>
+
+    <name>${project.artifactId}</name>
+    <description>WebAssembly Python host for Trino</description>
+    <url>https://trino.io</url>
+
+    <inceptionYear>2024</inceptionYear>
+
+    <licenses>
+        <license>
+            <name>Apache License 2.0</name>
+            <url>https://www.apache.org/licenses/LICENSE-2.0</url>
+            <distribution>repo</distribution>
+        </license>
+    </licenses>
+
+    <scm>
+        <connection>scm:git:git://github.com/trinodb/trino-wasm-python.git</connection>
+        <developerConnection>scm:git:[email protected]:trinodb/trino-wasm-python.git</developerConnection>
+        <tag>HEAD</tag>
+        <url>https://github.com/trinodb/trino-wasm-python</url>
+    </scm>
+
+    <properties>
+        <project.build.targetJdk>21</project.build.targetJdk>
+        <air.maven.version>3.8.8</air.maven.version>
+        <air.check.skip-spotbugs>true</air.check.skip-spotbugs>
+        <air.check.skip-pmd>true</air.check.skip-pmd>
+        <air.check.skip-jacoco>true</air.check.skip-jacoco>
+        <air.javadoc.lint>all,-missing</air.javadoc.lint>
+        <air.release.preparation-goals>clean verify -DskipTests</air.release.preparation-goals>
+
+        <dep.chicory.version>1.0.0-M2</dep.chicory.version>
+    </properties>
+
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>com.dylibso.chicory</groupId>
+                <artifactId>bom</artifactId>
+                <version>1.0.0-M2</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.dylibso.chicory</groupId>
+            <artifactId>runtime</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.dylibso.chicory</groupId>
+            <artifactId>wasm</artifactId>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <pluginManagement>
+            <plugins>
+                <plugin>
+                    <groupId>org.apache.maven.plugins</groupId>
+                    <artifactId>maven-surefire-plugin</artifactId>
+                    <configuration>
+                        <failIfNoTests>false</failIfNoTests>
+                    </configuration>
+                </plugin>
+            </plugins>
+        </pluginManagement>
+
+        <plugins>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>exec-maven-plugin</artifactId>
+                <version>3.5.0</version>
+                <configuration>
+                    <executable>${project.basedir}/build.sh</executable>
+                    <arguments>
+                        <argument>${wasmDebug}</argument>
+                    </arguments>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>build-wasm</id>
+                        <goals>
+                            <goal>exec</goal>
+                        </goals>
+                        <phase>generate-sources</phase>
+                    </execution>
+                </executions>
+            </plugin>
+
+            <plugin>
+                <groupId>com.dylibso.chicory</groupId>
+                <artifactId>aot-maven-plugin-experimental</artifactId>
+                <version>${dep.chicory.version}</version>
+                <executions>
+                    <execution>
+                        <id>aot-python</id>
+                        <goals>
+                            <goal>wasm-aot-gen</goal>
+                        </goals>
+                        <configuration>
+                            <name>io.trino.wasm.python.Python</name>
+                            <wasmFile>${project.build.directory}/wasm/python-host-opt.wasm</wasmFile>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+    <profiles>
+        <profile>
+            <id>wasm-debug</id>
+            <properties>
+                <wasmDebug>--debug</wasmDebug>
+            </properties>
+        </profile>
+    </profiles>
+</project>