[tools/sgx,curl] Minimize and build libcurl statically

Previously, we built `libsgx_util` as a shared library which was linked into RA-TLS, Secret Provisioning libraries and some other tools. This library relied on libcurl taken from the host platform. However, libcurl has a high number of dependencies itself, most of which are not required by Gramine, only consume space and are hard to track and reason about. This is especially pronounced for the RA-TLS/SecretProv libs because they typically run inside SGX enclaves and therefore should be minimal and reproducible. This patch: - builds `libsgx_util` into a static library, - builds and minimizes `libcurl` into a static library (leveraging `mbedtls` as the TLS backend), - links all dependencies of `tools/sgx` statically to avoid pulling in a lot of dependencies. Co-authored-by: Wojtek Porczyk <[email protected]> Signed-off-by: Kailun Qin <[email protected]> Signed-off-by: Wojtek Porczyk <[email protected]>
gramineproject · Nov 1, 2022 · c15ca88 · c15ca88
1 parent cda5d4d
commit c15ca88
Show file tree

Hide file tree

Showing 16 changed files with 200 additions and 26 deletions.
diff --git a/.ci/lib/stage-build-nosgx.jenkinsfile b/.ci/lib/stage-build-nosgx.jenkinsfile
@@ -45,7 +45,7 @@ stage('build') {
     env.GRAMINE_PKGLIBDIR = libdir + '/gramine'
 
     // In CI we install to non-standard --prefix (see above). This makes sure the libraries are
-    // available anyway (e.g. gramine-sgx-pf-crypt needs libsgx_util.so).
+    // available anyway.
     env.PKG_CONFIG_PATH = libdir + '/pkgconfig'
 
     // prevent cheating and testing from repo

diff --git a/.ci/lib/stage-build-sgx.jenkinsfile b/.ci/lib/stage-build-sgx.jenkinsfile
@@ -57,7 +57,7 @@ stage('build') {
     env.GRAMINE_PKGLIBDIR = libdir + '/gramine'
 
     // In CI we install to non-standard --prefix (see above). This makes sure the libraries are
-    // available anyway (e.g. gramine-sgx-pf-crypt needs libsgx_util.so).
+    // available anyway.
     env.PKG_CONFIG_PATH = libdir + '/pkgconfig'
 
     // prevent cheating and testing from repo

diff --git a/.ci/ubuntu18.04.dockerfile b/.ci/ubuntu18.04.dockerfile
@@ -17,7 +17,6 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install -y \
     jq \
     libapr1-dev \
     libaprutil1-dev \
-    libcurl4-openssl-dev \
     libelf-dev \
     libevent-dev \
     libexpat1 \

diff --git a/.ci/ubuntu20.04.dockerfile b/.ci/ubuntu20.04.dockerfile
@@ -18,7 +18,6 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install -y \
     libapr1-dev \
     libaprutil1-dev \
     libcjson-dev \
-    libcurl4-openssl-dev \
     libelf-dev \
     libevent-dev \
     libexpat1 \

diff --git a/CI-Examples/ra-tls-secret-prov/Makefile b/CI-Examples/ra-tls-secret-prov/Makefile
@@ -49,12 +49,12 @@ ssl/server.crt: ssl/ca_config.conf
 ######################### CLIENT/SERVER EXECUTABLES ###########################
 
 # Use hard-coded GRAMINEDIR because we currently fail to provide secret prov headers in Gramine
-# installation. We also use `sgx_util` pkg-config because we don't have a secret prov one.
+# installation. We also use `mbedtls_gramine` pkg-config because we don't have a secret prov one.
 # TODO: Create a pkg-config file for secretprov_gramine libs, and use it in below
 #       CFLAGS/LDFLAGS lines (via `pkg-config {--cflags|--libs} secretprov_gramine`).
 GRAMINEDIR ?= ../..
 CFLAGS += -Wall -std=c11 -I$(GRAMINEDIR)/tools/sgx/ra-tls
-LDFLAGS += -Wl,--enable-new-dtags $(shell pkg-config --libs sgx_util)
+LDFLAGS += -Wl,--enable-new-dtags $(shell pkg-config --libs mbedtls_gramine)
 
 %/server_epid: %/server.c
 	$(CC) $< $(CFLAGS) $(LDFLAGS) -lsecret_prov_verify_epid -pthread -o $@

diff --git a/Documentation/devel/building.rst b/Documentation/devel/building.rst
@@ -71,9 +71,8 @@ running, and Intel SGX SDK/PSW/DCAP must be installed.
 """"""""""""""""""""
 Run the following commands on Ubuntu to install SGX-related dependencies::
 
-    sudo apt-get install -y libcurl4-openssl-dev \
-        libprotobuf-c-dev protobuf-c-compiler protobuf-compiler \
-        python3-cryptography python3-pip python3-protobuf
+    sudo apt-get install -y libprotobuf-c-dev protobuf-c-compiler \
+        protobuf-compiler python3-cryptography python3-pip python3-protobuf
 
 2. Install Linux kernel with patched FSGSBASE
 """""""""""""""""""""""""""""""""""""""""""""

diff --git a/meson.build b/meson.build
@@ -240,14 +240,20 @@ uthash_dep = subproject('uthash-2.1.0').get_variable('uthash_dep')
 mbedtls_proj = subproject('mbedtls-mbedtls-3.2.1')
 mbedtls_static_dep = mbedtls_proj.get_variable('mbedtls_static_dep')
 mbedtls_pal_dep = mbedtls_proj.get_variable('mbedtls_pal_dep')
+mbedtls_curl_dep = mbedtls_proj.get_variable('mbedtls_curl_dep')
+# Specify a Meson runtime-dependency based on `mbedtls_curl_dep` so that the curl subproject can
+# find and retrieve it, see:
+# https://mesonbuild.com/Reference-manual_builtin_meson.html#mesonoverride_dependency for details
+meson.override_dependency('mbedtls_curl', mbedtls_curl_dep)
 
 if sgx
     protoc_c_prog = find_program('protoc-c')
     protoc_prog = find_program('protoc')
 
     threads_dep = dependency('threads')
 
-    libcurl_dep = dependency('libcurl', version: '>=7.58.0')
+    curl_proj = subproject('curl-7.84.0')
+    libcurl_dep = curl_proj.get_variable('curl_minimal_dep')
 
     cjson_dep = dependency('cjson', required: false)
     if not cjson_dep.found()

diff --git a/subprojects/.gitignore b/subprojects/.gitignore
@@ -1,6 +1,7 @@
 /packagecache
 
 /cJSON-*/
+/curl-*/
 /gcc-*/
 /glibc-*/
 /mbedtls-*/

diff --git a/subprojects/curl-7.84.0.wrap b/subprojects/curl-7.84.0.wrap
@@ -0,0 +1,7 @@
+[wrap-file]
+directory = curl-7.84.0
+source_url = https://github.com/curl/curl/releases/download/curl-7_84_0/curl-7.84.0.tar.gz
+source_fallback_url = https://packages.gramineproject.io/distfiles/curl-7.84.0.tar.gz
+source_filename = curl-7.84.0.tar.gz
+source_hash = 3c6893d38d054d4e378267166858698899e9d87258e8ff1419d020c395384535
+patch_directory = curl-7.84.0
diff --git a/subprojects/packagefiles/curl-7.84.0/compile.sh b/subprojects/packagefiles/curl-7.84.0/compile.sh
@@ -0,0 +1,89 @@
+#!/bin/sh
+
+set -e
+
+log() {
+    echo "curl: $*"
+}
+
+CURRENT_SOURCE_DIR="$1"
+CURRENT_BUILD_DIR="$2"
+PRIVATE_DIR="$3"
+SUBPROJ_ROOT="$4"
+shift 4
+
+BUILD_LOG=$(realpath "$CURRENT_BUILD_DIR/curl-build.log")
+rm -f "$BUILD_LOG"
+
+log "see $BUILD_LOG for full build log"
+
+log "preparing sources..."
+
+rm -rf "$PRIVATE_DIR"
+cp -ar "$CURRENT_SOURCE_DIR" "$PRIVATE_DIR"
+
+(
+    cd "$PRIVATE_DIR"
+
+    log "running configure..."
+    # The list of configure options is selected based on:
+    # https://github.com/curl/curl/blob/curl-7_84_0/docs/INSTALL.md#reducing-size
+    ./configure                                     \
+        --disable-alt-svc                           \
+        --disable-ares                              \
+        --disable-cookies                           \
+        --disable-crypto-auth                       \
+        --disable-dateparse                         \
+        --disable-dict                              \
+        --disable-dnsshuffle                        \
+        --disable-doh                               \
+        --disable-file                              \
+        --disable-ftp                               \
+        --disable-get-easy-options                  \
+        --disable-gopher                            \
+        --disable-hsts                              \
+        --disable-http-auth                         \
+        --disable-imap                              \
+        --disable-ldap                              \
+        --disable-ldaps                             \
+        --disable-libcurl-option                    \
+        --disable-manual                            \
+        --disable-mqtt                              \
+        --disable-netrc                             \
+        --disable-ntlm-wb                           \
+        --disable-pop3                              \
+        --disable-progress-meter                    \
+        --disable-proxy                             \
+        --disable-pthreads                          \
+        --disable-rtsp                              \
+        --disable-shared                            \
+        --disable-smb                               \
+        --disable-smtp                              \
+        --disable-socketpair                        \
+        --disable-telnet                            \
+        --disable-tftp                              \
+        --disable-threaded-resolver                 \
+        --disable-tls-srp                           \
+        --disable-unix-sockets                      \
+        --disable-verbose                           \
+        --disable-versioned-symbols                 \
+        --with-mbedtls="$SUBPROJ_ROOT"/mbedtls-curl \
+        --without-brotli                            \
+        --without-libidn2                           \
+        --without-libpsl                            \
+        --without-librtmp                           \
+        --without-nghttp2                           \
+        --without-ngtcp2                            \
+        --without-zlib                              \
+        --without-zstd                              \
+        >>"$BUILD_LOG" 2>&1
+
+    log "running make..."
+
+    # The curl executable is not needed so we only build libcurl here.
+    cd lib; make -j"$(nproc)" >>"$BUILD_LOG" 2>&1
+)
+
+cp -r "$PRIVATE_DIR"/lib/.libs/* "$CURRENT_BUILD_DIR"/
+
+log "done"
diff --git a/subprojects/packagefiles/curl-7.84.0/dummy.c b/subprojects/packagefiles/curl-7.84.0/dummy.c
@@ -0,0 +1 @@
+void dummy() {}
diff --git a/subprojects/packagefiles/curl-7.84.0/meson.build b/subprojects/packagefiles/curl-7.84.0/meson.build
@@ -0,0 +1,28 @@
+project('curl', 'c', version: '7.84.0')
+
+mbedtls_gramine = dependency('mbedtls_curl')
+
+# This dummy target is needed, because custom_target(depends:) needs a "target", not a "dependency"
+mbedtls_dummy_target = static_library('dummy', 'dummy.c', dependencies: mbedtls_gramine)
+
+curl_libs_output = [
+    'libcurl.a',
+]
+
+curl = custom_target('curl',
+    command: [
+        find_program('compile.sh'),
+        '@CURRENT_SOURCE_DIR@',
+        meson.current_build_dir(),
+        '@PRIVATE_DIR@',
+        join_paths(meson.build_root(), 'subprojects'),
+    ],
+
+    depends: mbedtls_dummy_target,
+    output: curl_libs_output,
+)
+
+curl_minimal_dep = declare_dependency(
+    link_with: curl,
+    include_directories: 'include',
+)
diff --git a/subprojects/packagefiles/mbedtls/compile-curl.sh b/subprojects/packagefiles/mbedtls/compile-curl.sh
@@ -0,0 +1,25 @@
+#!/bin/sh
+
+set -ex
+
+CURRENT_SOURCE_DIR="$1"
+VENDOR_SOURCE_DIR="$2"
+CURRENT_BUILD_DIR="$3"
+PRIVATE_DIR="$4"
+SUBPROJ_ROOT="$5"
+shift 5
+
+rm -rf "$PRIVATE_DIR"
+
+cp -ar "$VENDOR_SOURCE_DIR" "$PRIVATE_DIR"
+cp "$CURRENT_SOURCE_DIR"/include/mbedtls/*.h "$PRIVATE_DIR"/include/mbedtls/
+patch -p1 --directory "$PRIVATE_DIR" <"$CURRENT_SOURCE_DIR"/gramine.patch
+patch -p1 --directory "$PRIVATE_DIR" <"$CURRENT_SOURCE_DIR"/fcntl.patch
+
+make -C "$PRIVATE_DIR" lib SUFFIX="''" install DESTDIR="$SUBPROJ_ROOT"/mbedtls-curl
+touch "$PRIVATE_DIR"/library/mbedtls-curl-dummy.h
+
+for output in $@
+do
+    cp -a "$PRIVATE_DIR"/library/"$(basename "$output")" "$output"
+done
diff --git a/subprojects/packagefiles/mbedtls/compile.sh b/subprojects/packagefiles/mbedtls/compile.sh
@@ -1,7 +1,6 @@
 #!/bin/sh
 
-set -x
-set -e
+set -ex
 
 CURRENT_SOURCE_DIR="$1"
 VENDOR_SOURCE_DIR="$2"

diff --git a/subprojects/packagefiles/mbedtls/meson.build b/subprojects/packagefiles/mbedtls/meson.build
@@ -82,6 +82,29 @@ mbedtls_pal_libs = custom_target('mbedtls_pal',
     build_by_default: true,
 )
 
+mbedtls_curl_libs = custom_target('mbedtls_curl',
+    command: [
+        find_program('compile-curl.sh'),
+        '@CURRENT_SOURCE_DIR@',
+        '@CURRENT_SOURCE_DIR@/mbedtls-mbedtls-3.2.1',
+        meson.current_build_dir(),
+        '@PRIVATE_DIR@',
+        join_paths(meson.build_root(), 'subprojects'),
+        '@OUTPUT@',
+    ],
+
+    input: ['mbedtls-mbedtls-3.2.1/Makefile', 'gramine.patch'],
+
+    output: [
+        'libmbedcrypto.a',
+        'libmbedtls.a',
+        'libmbedx509.a',
+        'mbedtls-curl-dummy.h'
+    ],
+
+    build_by_default: true,
+)
+
 mbedtls_inc = include_directories('include', 'mbedtls-mbedtls-3.2.1/include')
 
 mbedtls_static_dep = declare_dependency(
@@ -96,3 +119,14 @@ mbedtls_pal_dep = declare_dependency(
     include_directories: mbedtls_inc,
     compile_args: '-DMBEDTLS_CONFIG_FILE="mbedtls/config-pal.h"',
 )
+mbedtls_curl_dep = declare_dependency(
+    # HACK: Apparently Meson considers the `mbedtls_curl_libs` to be "not linkable", because it has
+    # multiple outputs; however, it allows picking the outputs one by one.
+    link_with: [mbedtls_curl_libs[0], mbedtls_curl_libs[1], mbedtls_curl_libs[2]],
+    # HACK: Use the generated empty ".h" file and propagate it as part of the inter-subproject
+    # dependency to enforce compile order, i.e., to make sure `mbedtls_curl_libs` are ready before
+    # `libcurl` is compiling.
+    sources: [mbedtls_curl_libs[3]],
+    include_directories: mbedtls_inc,
+    compile_args: '-DMBEDTLS_CONFIG_FILE="mbedtls/config-pal.h"',
+)
diff --git a/tools/sgx/common/meson.build b/tools/sgx/common/meson.build
@@ -1,4 +1,4 @@
-sgx_util = shared_library('sgx_util',
+sgx_util = static_library('sgx_util',
     'ias.c',
     'ias.h',
     'pf_util.c',
@@ -35,16 +35,3 @@ sgx_util_dep = declare_dependency(
         protected_files_inc,
     ],
 )
-
-pkgconfig.generate(
-    sgx_util,
-    libraries: [
-        '-Wl,-rpath,${libdir}',
-        sgx_util,
-    ],
-)
-
-meson.add_install_script('/bin/sh', '-c',
-    'ln -sf ../../../libsgx_util.so ' +
-    '"$MESON_INSTALL_DESTDIR_PREFIX"/@0@/gramine/runtime/glibc/'.format(
-        get_option('libdir')))