Add mariner build images (#832)

dotnet · Mar 30, 2023 · 2288685 · 2288685
1 parent c1673b9
commit 2288685
Show file tree

Hide file tree

Showing 15 changed files with 772 additions and 1 deletion.
diff --git a/eng/pipelines/dotnet-buildtools-prereqs-mariner-pr.yml b/eng/pipelines/dotnet-buildtools-prereqs-mariner-pr.yml
@@ -24,3 +24,6 @@ stages:
   parameters:
     internalProjectName: ${{ variables.internalProjectName }}
     publicProjectName: ${{ variables.publicProjectName }}
+    linuxAmdBuildJobTimeout: 360
+    customBuildInitSteps:
+    - template: /eng/pipelines/steps/install-cross-build-prereqs.yml
diff --git a/eng/pipelines/dotnet-buildtools-prereqs-mariner.yml b/eng/pipelines/dotnet-buildtools-prereqs-mariner.yml
@@ -24,3 +24,6 @@ stages:
   parameters:
     internalProjectName: ${{ variables.internalProjectName }}
     publicProjectName: ${{ variables.publicProjectName }}
+    linuxAmdBuildJobTimeout: 360
+    customBuildInitSteps:
+    - template: /eng/pipelines/steps/install-cross-build-prereqs.yml
diff --git a/src/cbl-mariner/1.0/helix/amd64/Dockerfile b/src/cbl-mariner/1.0/helix/amd64/Dockerfile
@@ -1,4 +1,4 @@
-FROM cblmariner.azurecr.io/base/core:1.0
+FROM mcr.microsoft.com/cbl-mariner/base/core:1.0
 
 # Install Helix Dependencies
 

diff --git a/src/cbl-mariner/2.0/cross/amd64-alpine/Dockerfile b/src/cbl-mariner/2.0/cross/amd64-alpine/Dockerfile
@@ -0,0 +1,10 @@
+FROM mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-crossdeps-builder-local AS builder
+ARG ROOTFS_DIR=/crossrootfs/x64
+
+RUN /scripts/eng/common/cross/build-rootfs.sh x64 alpine3.13 --skipunmount
+
+
+FROM mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-crossdeps-amd64-local
+ARG ROOTFS_DIR=/crossrootfs/x64
+
+COPY --from=builder $ROOTFS_DIR $ROOTFS_DIR
diff --git a/src/cbl-mariner/2.0/cross/amd64/Dockerfile b/src/cbl-mariner/2.0/cross/amd64/Dockerfile
@@ -0,0 +1,14 @@
+FROM mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-crossdeps-builder-local AS builder
+ARG ROOTFS_DIR=/crossrootfs/x64
+
+RUN /scripts/eng/common/cross/build-rootfs.sh x64 xenial --skipunmount
+
+
+FROM mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-crossdeps-amd64-local
+ARG ROOTFS_DIR=/crossrootfs/x64
+
+COPY --from=builder $ROOTFS_DIR $ROOTFS_DIR
+
+RUN LLVM_VERSION=12.0.1 LLVM_VERSION_MAJOR="${LLVM_VERSION%%.*}" && \
+    mkdir -p $ROOTFS_DIR/usr/lib/llvm-${LLVM_VERSION_MAJOR}/lib/clang/${LLVM_VERSION}/lib/linux/ && \
+    cp /usr/lib/clang/${LLVM_VERSION}/lib/linux/libclang_rt.profile-x86_64.a $ROOTFS_DIR/usr/lib/llvm-${LLVM_VERSION_MAJOR}/lib/clang/${LLVM_VERSION}/lib/linux/
diff --git a/src/cbl-mariner/2.0/cross/arm64-alpine/Dockerfile b/src/cbl-mariner/2.0/cross/arm64-alpine/Dockerfile
@@ -0,0 +1,10 @@
+FROM mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-crossdeps-builder-local AS builder
+ARG ROOTFS_DIR=/crossrootfs/arm64
+
+RUN /scripts/eng/common/cross/build-rootfs.sh arm64 alpine3.13 --skipunmount
+
+
+FROM mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-crossdeps-arm-local
+ARG ROOTFS_DIR=/crossrootfs/arm64
+
+COPY --from=builder $ROOTFS_DIR $ROOTFS_DIR
diff --git a/src/cbl-mariner/2.0/cross/arm64/Dockerfile b/src/cbl-mariner/2.0/cross/arm64/Dockerfile
@@ -0,0 +1,38 @@
+FROM mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-crossdeps-builder-local AS builder
+ARG ROOTFS_DIR=/crossrootfs/arm64
+
+RUN /scripts/eng/common/cross/build-rootfs.sh arm64 xenial --skipunmount
+
+# Build compiler-rt profile library for PGO instrumentation
+RUN mkdir compiler-rt_build && cd compiler-rt_build && \
+    BUILD_FLAGS="-v --sysroot=$ROOTFS_DIR" \
+    TARGET_TRIPLE=aarch64-linux-gnu && \
+    cmake ../llvm-project.src/compiler-rt \
+        -DCOMPILER_RT_BUILD_PROFILE=ON \
+        -DCOMPILER_RT_BUILD_BUILTINS=OFF \
+        -DCOMPILER_RT_BUILD_SANITIZERS=OFF \
+        -DCOMPILER_RT_BUILD_XRAY=OFF \
+        -DCOMPILER_RT_BUILD_LIBFUZZER=OFF \
+        \
+        -DCMAKE_C_COMPILER=clang \
+        -DCMAKE_CXX_COMPILER=clang++ \
+        -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld" \
+        -DCMAKE_C_COMPILER_TARGET=${TARGET_TRIPLE} \
+        -DCMAKE_CXX_COMPILER_TARGET=${TARGET_TRIPLE} \
+        -DLLVM_CONFIG_PATH=llvm-config \
+        -DCOMPILER_RT_DEFAULT_TARGET_ONLY=ON \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_INSTALL_PREFIX=$ROOTFS_DIR/usr \
+        -DCMAKE_C_FLAGS="${BUILD_FLAGS}" \
+        -DCMAKE_CXX_FLAGS="${BUILD_FLAGS}" && \
+    make -j $(getconf _NPROCESSORS_ONLN)
+
+RUN LLVM_VERSION=12.0.1 LLVM_VERSION_MAJOR="${LLVM_VERSION%%.*}" && \
+    mkdir -p $ROOTFS_DIR/usr/lib/llvm-${LLVM_VERSION_MAJOR}/lib/clang/${LLVM_VERSION}/lib/linux/ && \
+    cp compiler-rt_build/lib/linux/libclang_rt.profile-aarch64.a $ROOTFS_DIR/usr/lib/llvm-${LLVM_VERSION_MAJOR}/lib/clang/${LLVM_VERSION}/lib/linux/
+
+
+FROM mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-crossdeps-arm-local
+ARG ROOTFS_DIR=/crossrootfs/arm64
+
+COPY --from=builder $ROOTFS_DIR $ROOTFS_DIR
diff --git a/src/cbl-mariner/2.0/cross/x86/Dockerfile b/src/cbl-mariner/2.0/cross/x86/Dockerfile
@@ -0,0 +1,43 @@
+FROM mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-crossdeps-builder-local AS builder
+ARG ROOTFS_DIR=/crossrootfs/x86
+
+# Provides LLVMConfig.cmake, needed to build compiler-rt. Only needed on x86.
+# x64 doesn't need this because we use compiler-rt from the mariner package.
+# arm/arm64 don't need this because LLVMConfig.cmake is provided by the LLVM we build from source.
+RUN tdnf install -y llvm-devel
+
+RUN /scripts/eng/common/cross/build-rootfs.sh x86 xenial --skipunmount
+
+# Build compiler-rt profile library for PGO instrumentation
+RUN mkdir compiler-rt_build && cd compiler-rt_build && \
+    BUILD_FLAGS="-v --sysroot=$ROOTFS_DIR" \
+    TARGET_TRIPLE=i386-linux-gnu && \
+    cmake ../llvm-project.src/compiler-rt \
+        -DCOMPILER_RT_BUILD_PROFILE=ON \
+        -DCOMPILER_RT_BUILD_BUILTINS=OFF \
+        -DCOMPILER_RT_BUILD_SANITIZERS=OFF \
+        -DCOMPILER_RT_BUILD_XRAY=OFF \
+        -DCOMPILER_RT_BUILD_LIBFUZZER=OFF \
+        \
+        -DCMAKE_C_COMPILER=clang \
+        -DCMAKE_CXX_COMPILER=clang++ \
+        -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld" \
+        -DCMAKE_C_COMPILER_TARGET=${TARGET_TRIPLE} \
+        -DCMAKE_CXX_COMPILER_TARGET=${TARGET_TRIPLE} \
+        -DLLVM_CONFIG_PATH=llvm-config \
+        -DCOMPILER_RT_DEFAULT_TARGET_ONLY=ON \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_INSTALL_PREFIX=$ROOTFS_DIR/usr \
+        -DCMAKE_C_FLAGS="${BUILD_FLAGS}" \
+        -DCMAKE_CXX_FLAGS="${BUILD_FLAGS}" && \
+    make -j $(getconf _NPROCESSORS_ONLN)
+
+RUN LLVM_VERSION=12.0.1 LLVM_VERSION_MAJOR="${LLVM_VERSION%%.*}" && \
+    mkdir -p $ROOTFS_DIR/usr/lib/llvm-${LLVM_VERSION_MAJOR}/lib/clang/${LLVM_VERSION}/lib/linux/ && \
+    cp compiler-rt_build/lib/linux/libclang_rt.profile-i386.a $ROOTFS_DIR/usr/lib/llvm-${LLVM_VERSION_MAJOR}/lib/clang/${LLVM_VERSION}/lib/linux/
+
+
+FROM mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-crossdeps-amd64-local
+ARG ROOTFS_DIR=/crossrootfs/x86
+
+COPY --from=builder $ROOTFS_DIR $ROOTFS_DIR
diff --git a/src/cbl-mariner/2.0/crossdeps-amd64/Dockerfile b/src/cbl-mariner/2.0/crossdeps-amd64/Dockerfile
@@ -0,0 +1,6 @@
+FROM mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-crossdeps-local
+
+RUN tdnf install -y \
+        clang \
+        lld \
+        compiler-rt
diff --git a/src/cbl-mariner/2.0/crossdeps-arm/Dockerfile b/src/cbl-mariner/2.0/crossdeps-arm/Dockerfile
@@ -0,0 +1,6 @@
+FROM mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-crossdeps-builder-local AS builder
+
+FROM mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-crossdeps-local
+
+# Install LLVM that we built from source
+COPY --from=builder /usr/local /usr/local
diff --git a/src/cbl-mariner/2.0/crossdeps-builder/Dockerfile b/src/cbl-mariner/2.0/crossdeps-builder/Dockerfile
@@ -0,0 +1,64 @@
+FROM mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-crossdeps-local
+
+# TODO: remove this once debootstrap is available in the base repos.
+COPY mariner-extended.repo /etc/yum.repos.d/
+
+RUN tdnf install -y \
+        # Rootfs build dependencies
+        debootstrap \
+        # LLVM build dependencies
+        texinfo \
+        diffutils \
+        binutils
+
+# Obtain ubuntu package signing key (for use by debootstrap)
+# 1. Add public key used to sign the ubuntu keyring
+COPY dimitri_john_ledkov.asc .
+RUN gpg --output dimitri_john_ledkov.gpg --dearmor dimitri_john_ledkov.asc && \
+    rm dimitri_john_ledkov.asc && \
+# 2. Download the ubuntu keyrings
+    wget https://mirrors.edge.kernel.org/ubuntu/pool/main/u/ubuntu-keyring/ubuntu-keyring_2021.03.26.tar.gz && \
+    echo "492eed5c06408c6f632577adb0796130af5d6542013ef418f47187a209e49bb1 ubuntu-keyring_2021.03.26.tar.gz" | sha256sum -c && \
+    tar xf ubuntu-keyring_2021.03.26.tar.gz && \
+    rm ubuntu-keyring_2021.03.26.tar.gz && \
+# 3. Verify keyrings
+    pushd ubuntu-keyring-2021.03.26 && \
+    gpg --keyring /dimitri_john_ledkov.gpg --output SHA512SUMS.txt --decrypt SHA512SUMS.txt.asc && \
+    rm /dimitri_john_ledkov.gpg && \
+    sha512sum -c SHA512SUMS.txt && \
+# 4. Install the needed keyring and delete the rest
+    mkdir -p /usr/share/keyrings && \
+    mv keyrings/ubuntu-archive-keyring.gpg /usr/share/keyrings && \
+    popd && \
+    rm -r ubuntu-keyring-2021.03.26
+
+# 1. Obtain signing keys used to sign llvm sources
+RUN wget https://releases.llvm.org/release-keys.asc && \
+    echo "f181a90697e3ea4b7782f1ee48314a570aef058505b4f3a0ab0611094ec13241 release-keys.asc" | sha256sum -c && \
+    gpg --output release-keys.gpg --dearmor release-keys.asc && \
+    rm release-keys.asc && \
+# 2. Download llvm sources and signature, and verify signature
+    LLVM_VERSION=12.0.1 && \
+    wget -O llvm-project.src.tar.xz.sig https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/llvm-project-${LLVM_VERSION}.src.tar.xz.sig && \
+    echo "6cc956d622a7d3d746de0d71d8ca616a6c291e2c561703ac7a9535f38b999955 llvm-project.src.tar.xz.sig" | sha256sum -c && \
+    wget -O llvm-project.src.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/llvm-project-${LLVM_VERSION}.src.tar.xz && \
+    echo "129cb25cd13677aad951ce5c2deb0fe4afc1e9d98950f53b51bdcfb5a73afa0e llvm-project.src.tar.xz" | sha256sum -c && \
+    gpg --keyring /release-keys.gpg --verify llvm-project.src.tar.xz.sig && \
+    rm llvm-project.src.tar.xz.sig
+
+# Build LLVM cross-toolchain (with support for targeting arm architectures)
+RUN mkdir llvm-project.src && \
+    tar -xf llvm-project.src.tar.xz --directory llvm-project.src --strip-components=1 && \
+    rm llvm-project.src.tar.xz && \
+    mkdir build && cd build && \
+    cmake ../llvm-project.src/llvm \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DLLVM_TARGETS_TO_BUILD="host;AArch64;ARM" \
+        -Wno-dev \
+        -DLLVM_ENABLE_PROJECTS="clang;lld" && \
+    make -j $(getconf _NPROCESSORS_ONLN) && \
+    make install
+
+# Obtain arcade scripts used to build rootfs
+RUN git config --global user.email builder@dotnet-buildtools-prereqs-docker && \
+    git clone --depth 1 --single-branch https://github.com/dotnet/arcade /scripts