Add a build matrix for Ruby with ASAN

It compiles some of Ruby's key dependencies with ASAN enabled, and
statically links them. It builds with Clang 18, which is the minimum
required for ASAN to work.

.github/workflows/build.yml

@@ -80,6 +80,9 @@ jobs:
       matrix:
         os: [ ubuntu-20.04, ubuntu-22.04, ubuntu-24.04, macos-11, macos-arm-oss ]
         name: [ head, debug ]
+        include:
+          - os: ubuntu-22.04
+            name: asan
     runs-on: ${{ matrix.os }}
     steps:
     - name: Clone ruby
@@ -129,6 +132,28 @@ jobs:
         echo "optflags=-O3 -fno-inline" >> $GITHUB_ENV
       if: matrix.name == 'debug'
 
+    - name: Install Clang 18
+      run: |
+        set -ex;
+        sudo add-apt-repository 'deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main'
+        sudo apt update
+        sudo apt install -y clang-18
+    - name: Compile dependencies with ASAN
+      run: './asan_libs.sh'
+      if: matrix.name == 'asan'
+    - name: Set configure flags (asan)
+      run: |
+        echo "optflags=-O3 -fno-inline" >> $GITHUB_ENV
+        # Compile with ASAN
+        echo "cflags=-fsanitize=address -fno-omit-frame-poiner -ggdb3" >> $GITHUB_ENV
+        # Requires clang 18
+        echo "CC=clang-18"
+        # Clang > 17 does not work with M:N threading: https://bugs.ruby-lang.org/issues/20243
+        echo "cppflags=-DENABLE_PATH_CHECK=0 -DRUBY_DEBUG=1 -DVM_CHECK_MODE=1 -DUSE_MN_THREADS=0" >> $GITHUB_ENV
+        # And statically link against our ASAN-compiled libraries
+        echo "LDFLAGS=-L$(realpath third_party_asan/lib/) -Wl,-Bstatic -lcrypto -lssl -lyaml -lffi -Wl,-Bdynamic" >> $GITHUB_ENV
+      if: matrix.name == 'asan'
+
     # Build
     - run: chmod 755 $HOME # https://github.com/actions/virtual-environments/issues/267
     - run: mkdir -p ~/.rubies

asan_libs.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -ex;
+
+RUBY_SRCDIR="$(realpath .)"
+ASAN_FLAGS="-fsanitize=address -fPIC -ggdb3"
+OPENSSL_VERSION="3.3.0"
+LIBYAML_VERSION="0.2.5"
+LIBFFI_VERSION="3.4.5"
+
+mkdir -p third_party_asan/{build,lib}
+
+cd "$RUBY_SRCDIR/third_party_asan/build"
+wget "https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz"
+tar xf "openssl-${OPENSSL_VERSION}.tar.gz"
+cd "openssl-${OPENSSL_VERSION}"
+./Configure CC=clang CFLAGS="$ASAN_FLAGS" no-shared no-tests no-apps
+make
+cp libcrypto.a libssl.a "$RUBY_SRCDIR/third_party_asan/lib"
+
+cd "$RUBY_SRCDIR/third_party_asan/build"
+wget "http://pyyaml.org/download/libyaml/yaml-${LIBYAML_VERSION}.tar.gz"
+tar xf "yaml-${LIBYAML_VERSION}.tar.gz"
+cd "yaml-${LIBYAML_VERSION}"
+./configure CC=clang CFLAGS="$ASAN_FLAGS" --disable-shared --enable-static
+make
+cp src/.libs/libyaml.a "$RUBY_SRCDIR/third_party_asan/lib"
+
+cd "$RUBY_SRCDIR/third_party_asan/build"
+wget "https://github.com/libffi/libffi/releases/download/v${LIBFFI_VERSION}/libffi-${LIBFFI_VERSION}.tar.gz"
+tar xf "libffi-${LIBFFI_VERSION}.tar.gz"
+cd "libffi-${LIBFFI_VERSION}"
+./configure CC=clang CFLAGS="$ASAN_FLAGS" --disable-shared --enable-static
+make
+cp "$(sh ./config.guess)/.libs/libffi.a" "$RUBY_SRCDIR/third_party_asan/lib"