use docker image for build

.github/workflows/build_and_test.yml

@@ -148,6 +148,7 @@ jobs:
   build-and-test-pglite-dependents:
     name: Build and Test packages dependent on PGlite
     runs-on: ubuntu-22.04
+
     needs: [build-and-test-pglite]
     steps:
       - uses: actions/checkout@v4

.github/workflows/build_wasm_postgres.yml

@@ -12,20 +12,22 @@ jobs:
   build:
     name: Build WASM Postgres
     runs-on: ubuntu-22.04
-    env:
-      BUILD_CONFIG_FILE: .buildconfig
-      SDK_ARCHIVE: python3.13-wasm-sdk-Ubuntu-22.04.tar.lz4
-      WASI_SDK_ARCHIVE: python3.13-wasi-sdk-Ubuntu-22.04.tar.lz4
-      SDKROOT: /opt/python-wasm-sdk
-      SYS_PYTHON: /usr/bin/python3
-      PGROOT: /tmp/pglite
-      TOTAL_MEMORY: 192MB
-      CMA_MB: 16
-      OBJDUMP: true
-      contrib: contrib
-      extra: extra
-      EXTRA_EXT: vector postgis
-      DEBUG: false
+    container:
+      image: electricsql/pglite-builder:REL_16_STABLE_3.1.74.2bi
+      env:
+        BUILD_CONFIG_FILE: .buildconfig
+        SDK_ARCHIVE: python3.13-wasm-sdk-Ubuntu-22.04.tar.lz4
+        WASI_SDK_ARCHIVE: python3.13-wasi-sdk-Ubuntu-22.04.tar.lz4
+        SDKROOT: /opt/python-wasm-sdk
+        SYS_PYTHON: /usr/bin/python3
+        PGROOT: /tmp/pglite
+        TOTAL_MEMORY: 128MB
+        CMA_MB: 64
+        OBJDUMP: true
+        contrib: contrib
+        extra: extra
+        EXTRA_EXT: vector
+        DEBUG: false
 
     steps:
       - uses: actions/checkout@v4
@@ -53,7 +55,7 @@ jobs:
         if: steps.cache-restore.outputs.cache-hit != 'true'
         working-directory: /
         run: |
-          sudo apt-get install -y lz4 wget pv bash
+          apt update && apt-get install -y lz4 wget pv bash curl git build-essential libreadline-dev zlib1g-dev bison flex file
           echo https://github.com/pygame-web/python-wasm-sdk/releases/download/$SDK_VERSION/$SDK_ARCHIVE
           curl -sL --retry 5 https://github.com/pygame-web/python-wasm-sdk/releases/download/$SDK_VERSION/$SDK_ARCHIVE | tar xvP --use-compress-program=lz4 | pv -p -l -s 34500 >/dev/null
           echo https://github.com/pygame-web/python-wasi-sdk/releases/download/$WASI_SDK_VERSION/$WASI_SDK_ARCHIVE

README.md

@@ -175,6 +175,31 @@ pnpm changeset
 ```
 And follow the instructions to create an appropriate changeset. Please ensure any contributions that touch code are accompanied by a changeset.
 
+### Building everything with Docker
+
+You can also build the PGlite from scratch by just issuing a docker build command:
+
+`$ docker build -f buildall.dockerfile -t pglitedocker .`
+
+This will build an image containing both the wasm and typescript parts of pglite.
+
+You can then use this image, for example, to run the tests:
+
+`$ docker run pglitedocker bash -c "cd /workspace/packages/pglite && pnpm test"`
+
+#### Extract build artifacts from the docker image
+
+Following is an example of how you can extract the build artifacts created in the previous step:
+
+```
+$ container_id=$(docker create pglitedocker)
+$ docker cp "$container_id:/tmp/pglite/sdk" /tmp/
+$ docker cp "$container_id:/tmp/web/postgres.wasm" /tmp/
+$ docker rm "$container_id"
+```
+
+This will copy the sdk build artifacts folder as well as the postgres.wasm binary to your local /tmp folder
+
 ## Acknowledgments
 
 PGlite builds on the work of [Stas Kelvich](https://github.com/kelvich) of [Neon](https://neon.tech) in this [Postgres fork](https://github.com/electric-sql/postgres-wasm).

buildall.dockerfile

@@ -0,0 +1,26 @@
+ARG PG_VERSION=git
+ARG SDK_VERSION=3.1.74.2bi
+FROM electricsql/pglite-builder:${PG_VERSION}_${SDK_VERSION}
+
+ENV NODE_VERSION=20.18.1
+RUN apt update && apt install -y curl git build-essential libreadline-dev zlib1g-dev bison flex file
+RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash
+ENV NVM_DIR=/root/.nvm
+RUN . "$NVM_DIR/nvm.sh" && nvm install ${NODE_VERSION}
+RUN . "$NVM_DIR/nvm.sh" && nvm use v${NODE_VERSION}
+RUN . "$NVM_DIR/nvm.sh" && nvm alias default v${NODE_VERSION}
+ENV PATH="/root/.nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}"
+RUN node --version
+RUN npm --version
+RUN corepack enable pnpm
+
+# RUN wget -qO- https://get.pnpm.io/install.sh | ENV="$HOME/.bashrc" SHELL="$(which bash)" bash -
+
+WORKDIR /workspace
+
+COPY . .
+
+RUN pnpm install
+# runs the wasm build without spinning yet another docker instance that would produce them
+RUN pnpm wasm:build-no-docker
+RUN pnpm ts:build

cibuild.sh

@@ -419,26 +419,26 @@ then
 
 fi
 
- if echo " $*"|grep -q " extra"
+if echo " $*"|grep -q " extra"
 then
     for extra_ext in  ${EXTRA_EXT:-"vector"}
     do
-        if $CI
-        then
-            if [ -d $PREFIX/include/X11 ]
-            then
-                echo -n
-            else
-                # install EXTRA sdk
-                . /etc/lsb-release
-                DISTRIB="${DISTRIB_ID}-${DISTRIB_RELEASE}"
-                CIVER=${CIVER:-$DISTRIB}
-                SDK_URL=https://github.com/pygame-web/python-wasm-sdk-extra/releases/download/$SDK_VERSION/python3.13-emsdk-sdk-extra-${CIVER}.tar.lz4
-                echo "Installing $SDK_URL"
-                curl -sL --retry 5 $SDK_URL | tar xvP --use-compress-program=lz4 | pv -p -l -s 15000 >/dev/null
-                chmod +x ./extra/*.sh
-            fi
-        fi
+        # if $CI
+        # then
+        #     if [ -d $PREFIX/include/X11 ]
+        #     then
+        #         echo -n
+        #     else
+        #         # install EXTRA sdk
+        #         . /etc/lsb-release
+        #         DISTRIB="${DISTRIB_ID}-${DISTRIB_RELEASE}"
+        #         CIVER=${CIVER:-$DISTRIB}
+        #         SDK_URL=https://github.com/pygame-web/python-wasm-sdk-extra/releases/download/$SDK_VERSION/python3.13-emsdk-sdk-extra-${CIVER}.tar.lz4
+        #         echo "Installing $SDK_URL"
+        #         curl -sL --retry 5 $SDK_URL | tar xvP --use-compress-program=lz4 | pv -p -l -s 15000 >/dev/null
+        #         chmod +x ./extra/*.sh
+        #     fi
+        # fi
         echo "======================= ${extra_ext} : $(pwd) ==================="
 
         ./extra/${extra_ext}.sh || exit 400
@@ -590,7 +590,7 @@ ________________________________________________________________________________
 
         pglite-prep) echo "==================== pglite-prep  =========================="
             mkdir -p $PGLITE/release
-            #rm $PGLITE/release/*
+            # rm -r $PGLITE/release/*
 
             # copy packed extensions
             cp -vf ${WEBROOT}/*.tar.gz ${PGLITE}/release/

cibuild/build-all.sh

@@ -1,5 +1,10 @@
 #!/bin/bash
 echo "======== build-all.sh : $(pwd)             =========="
+
+echo "======== Installing packages =========="
+
+apt update && apt install -y git bison flex
+
 echo "======== Building all PGlite prerequisites =========="
 
 # move copy of patches into dir

cibuild/build-without-docker.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+echo "======== build-without-docker.sh : $(pwd)                 =========="
+echo "======== Building all PGlite prerequisites using Docker =========="
+
+trap 'echo caught interrupt and exiting;' INT
+
+source .buildconfig
+
+if [[ -z "$SDK_VERSION" || -z "$PG_VERSION" ]]; then
+  echo "Missing SDK_VERSION and PG_VERSION env vars."
+  echo "Source them from .buildconfig"
+  exit 1
+fi
+
+export IMG_NAME="electricsql/pglite-builder"
+export IMG_TAG="${PG_VERSION}_${SDK_VERSION}"
+export SDK_ARCHIVE=python3.13-wasm-sdk-Ubuntu-22.04.tar.lz4
+export WASI_SDK_ARCHIVE=python3.13-wasi-sdk-Ubuntu-22.04.tar.lz4
+export OBJDUMP=${OBJDUMP:-true}
+
+./cibuild/build-all.sh

cibuild/pgbuild.sh

@@ -46,13 +46,14 @@ CC_PGLITE=$CC_PGLITE
         export MAIN_MODULE=""
     else
         # --with-libxml does not fit with --without-zlib
-        if $CI
-        then
-            # do not build obsolete ext xml2 on CI
-            XML2="--with-zlib --with-libxml"
-        else
-            XML2="--with-zlib --with-libxml --with-libxslt"
-        fi
+        # if $CI
+        # then
+        #     # do not build obsolete ext xml2 on CI
+        #     XML2="--with-zlib --with-libxml"
+        # else
+        #     XML2="--with-zlib --with-libxml --with-libxslt"
+        # fi
+        XML2="--with-zlib --with-libxml --with-libxslt"
         UUID="--with-uuid=ossp"
         BUILD=emscripten
         WASM_CFLAGS=""

package.json

@@ -12,7 +12,9 @@
     "ci:publish": "pnpm changeset publish",
     "build:all": "pnpm wasm:build && pnpm ts:build",
     "ts:build": "pnpm -r --filter \"./packages/**\" build",
-    "wasm:build": "bash ./cibuild/build-with-docker.sh"
+    "wasm:build": "bash ./cibuild/build-with-docker.sh",
+    "wasm:build-no-docker": "bash ./cibuild/build-without-docker.sh",
+    "docker:buildpglite": "docker build -f buildall.dockerfile ."
   },
   "devDependencies": {
     "@changesets/cli": "^2.27.9",