Use prebuilt base docker images for CI (#5327)

* Use prebuilt base docker images for CI

* bump Catch dependency

* code review feedback

CHANGELOG.md

@@ -17,6 +17,7 @@
 -----------
 
 ### Internals
+* Catch2 was updated to 2.13.8. ([#5327](https://github.com/realm/realm-core/pull/5327))
 * Mutating a committed MutableSubscriptionSet will throw a LogicError. ([#5162](https://github.com/realm/realm-core/pull/5162))
 
 ----------------------------------------------

Jenkinsfile

@@ -79,7 +79,8 @@ jobWrapper {
         stage('FormatCheck') {
             rlmNode('docker') {
                 getArchive()
-                docker.build('realm-core-clang:snapshot', '-f clang.Dockerfile .').inside() {
+
+                buildDockerEnv('testing.Dockerfile').inside {
                     echo "Checking code formatting"
                     modifications = sh(returnStdout: true, script: "git clang-format --diff ${targetSHA1}").trim()
                     try {
@@ -255,16 +256,17 @@ def doCheckInDocker(Map options = [:]) {
     return {
         rlmNode('docker') {
             getArchive()
-            def sourcesDir = pwd()
-            def buildEnv = docker.build 'realm-core-linux:21.04'
+
+            def buildEnv = buildDockerEnv('testing.Dockerfile')
+
             def environment = environment()
             if (options.useEncryption) {
                 environment << 'UNITTEST_ENCRYPT_ALL=1'
             }
 
             def buildSteps = { String dockerArgs = "" ->
-                withEnv(environment) {
-                    buildEnv.inside(dockerArgs) {
+                buildEnv.inside(dockerArgs) {
+                    withEnv(environment) {
                         try {
                             dir('build-dir') {
                                 sh "cmake ${cmakeDefinitions} -G Ninja .."
@@ -345,10 +347,13 @@ def doCheckSanity(Map options = [:]) {
     return {
         rlmNode('docker') {
             getArchive()
-            def buildEnv = docker.build('realm-core-linux:clang', '-f clang.Dockerfile .')
-            def environment = environment()
-            withEnv(environment) {
-                buildEnv.inside(privileged) {
+
+            def environment = environment() + [
+              'CC=clang',
+              'CXX=clang++'
+            ]
+            buildDockerEnv('testing.Dockerfile').inside(privileged) {
+                withEnv(environment) {
                     try {
                         dir('build-dir') {
                             sh "cmake ${cmakeDefinitions} -G Ninja .."
@@ -375,12 +380,12 @@ def doBuildLinux(String buildType) {
         rlmNode('docker') {
             getSourceArchive()
 
-            docker.build('realm-core-generic:gcc-11', '-f generic.Dockerfile .').inside {
+            buildDockerEnv('packaging.Dockerfile').inside {
                 sh """
                    rm -rf build-dir
                    mkdir build-dir
                    cd build-dir
-                   scl enable devtoolset-11 -- cmake -DCMAKE_BUILD_TYPE=${buildType} -DREALM_NO_TESTS=1 -G Ninja ..
+                   cmake -DCMAKE_BUILD_TYPE=${buildType} -DREALM_NO_TESTS=1 -G Ninja ..
                    ninja
                    cpack -G TGZ
                 """
@@ -400,18 +405,27 @@ def doBuildLinuxClang(String buildType) {
     return {
         rlmNode('docker') {
             getArchive()
-            docker.build('realm-core-linux:clang', '-f clang.Dockerfile .').inside() {
-                dir('build-dir') {
-                    sh "cmake -D CMAKE_BUILD_TYPE=${buildType} -DREALM_NO_TESTS=1 -G Ninja .."
-                    runAndCollectWarnings(
-                        script: 'ninja',
-                        parser: "clang",
-                        name: "linux-clang-${buildType}",
-                        filters: warningFilters,
-                    )
-                    sh 'cpack -G TGZ'
+
+            def environment = environment() + [
+              'CC=clang',
+              'CXX=clang++'
+            ]
+
+            buildDockerEnv('testing.Dockerfile').inside {
+                withEnv(environment) {
+                    dir('build-dir') {
+                        sh "cmake -D CMAKE_BUILD_TYPE=${buildType} -DREALM_NO_TESTS=1 -G Ninja .."
+                        runAndCollectWarnings(
+                            script: 'ninja',
+                            parser: "clang",
+                            name: "linux-clang-${buildType}",
+                            filters: warningFilters,
+                        )
+                        sh 'cpack -G TGZ'
+                    }
                 }
             }
+
             dir('build-dir') {
                 archiveArtifacts("*.tar.gz")
                 def stashName = "linux___${buildType}"
@@ -426,30 +440,30 @@ def doCheckValgrind() {
     return {
         rlmNode('docker') {
             getArchive()
-            def buildEnv = docker.build 'realm-core-linux:21.04'
+
             def environment = environment()
             environment << 'UNITTEST_NO_ERROR_EXITCODE=1'
-            withEnv(environment) {
-                buildEnv.inside {
-                    def workspace = pwd()
+
+            buildDockerEnv('testing.Dockerfile').inside {
+                withEnv(environment) {
                     try {
-                        sh """
+                        sh '''
                            mkdir build-dir
                            cd build-dir
                            cmake -D CMAKE_BUILD_TYPE=RelWithDebInfo -D REALM_VALGRIND=ON -D REALM_ENABLE_ALLOC_SET_ZERO=ON -D REALM_MAX_BPNODE_SIZE=1000 -G Ninja ..
-                        """
+                        '''
                         runAndCollectWarnings(
                             script: 'cd build-dir && ninja',
                             name: "linux-valgrind",
                             filters: warningFilters,
                         )
-                        sh """
+                        sh '''
                             cd build-dir/test
                             valgrind --version
-                            valgrind --tool=memcheck --leak-check=full --undef-value-errors=yes --track-origins=yes --child-silent-after-fork=no --trace-children=yes --suppressions=${workspace}/test/valgrind.suppress --error-exitcode=1 ./realm-tests
-                        """
+                            valgrind --tool=memcheck --leak-check=full --undef-value-errors=yes --track-origins=yes --child-silent-after-fork=no --trace-children=yes --suppressions=$WORKSPACE/test/valgrind.suppress --error-exitcode=1 ./realm-tests
+                        '''
                     } finally {
-                        recordTests("Linux-ValgrindDebug")
+                        recordTests('Linux-Valgrind')
                     }
                 }
             }
@@ -463,7 +477,9 @@ def doAndroidBuildInDocker(String abi, String buildType, TestAction test = TestA
             getArchive()
             def stashName = "android___${abi}___${buildType}"
             def buildDir = "build-${stashName}".replaceAll('___', '-')
-            def buildEnv = buildDockerEnv('ci/realm-core:android', extra_args: '-f android.Dockerfile', push: env.BRANCH_NAME == 'master')
+
+            def buildEnv = buildDockerEnv('android.Dockerfile')
+
             def environment = environment()
             def cmakeArgs = ''
             if (test == TestAction.None) {
@@ -685,7 +701,7 @@ def buildPerformance() {
       // REALM_BENCH_DIR tells the gen_bench_hist.sh script where to place results
       // REALM_BENCH_MACHID gives the results an id - results are organized by hardware to prevent mixing cached results with runs on different machines
       // MPLCONFIGDIR gives the python matplotlib library a config directory, otherwise it will try to make one on the user home dir which fails in docker
-      docker.build('realm-core-linux:21.04').inside {
+      buildDockerEnv('testing.Dockerfile').inside {
         withEnv(["REALM_BENCH_DIR=${env.WORKSPACE}/test/bench/core-benchmarks", "REALM_BENCH_MACHID=docker-brix","MPLCONFIGDIR=${env.WORKSPACE}/test/bench/config"]) {
           rlmS3Get file: 'core-benchmarks.zip', path: 'downloads/core/core-benchmarks.zip'
           sh 'unzip core-benchmarks.zip -d test/bench/'
@@ -860,28 +876,26 @@ def doBuildCoverage() {
   return {
     rlmNode('docker') {
       getArchive()
-      docker.build('realm-core-linux:21.04').inside {
-        def workspace = pwd()
-        sh """
+
+      buildDockerEnv('testing.Dockerfile').inside {
+        sh '''
           mkdir build
           cd build
           cmake -G Ninja -D REALM_COVERAGE=ON ..
           ninja
           cd ..
-          lcov --no-external --capture --initial --directory . --output-file ${workspace}/coverage-base.info
+          lcov --no-external --capture --initial --directory . --output-file $WORKSPACE/coverage-base.info
           cd build/test
           ulimit -c unlimited
           UNITTEST_PROGRESS=1 ./realm-tests
           cd ../..
-          lcov --no-external --directory . --capture --output-file ${workspace}/coverage-test.info
-          lcov --add-tracefile ${workspace}/coverage-base.info --add-tracefile coverage-test.info --output-file ${workspace}/coverage-total.info
-          lcov --remove ${workspace}/coverage-total.info '/usr/*' '${workspace}/test/*' --output-file ${workspace}/coverage-filtered.info
+          lcov --no-external --directory . --capture --output-file $WORKSPACE/coverage-test.info
+          lcov --add-tracefile $WORKSPACE/coverage-base.info --add-tracefile coverage-test.info --output-file $WORKSPACE/coverage-total.info
+          lcov --remove $WORKSPACE/coverage-total.info '/usr/*' '$WORKSPACE/test/*' --output-file $WORKSPACE/coverage-filtered.info
           rm coverage-base.info coverage-test.info coverage-total.info
-        """
+        '''
         withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-token-core', variable: 'CODECOV_TOKEN']]) {
-          sh '''
-            curl -s https://codecov.io/bash | bash
-          '''
+          sh 'curl -s https://codecov.io/bash | bash'
         }
       }
     }
@@ -948,6 +962,14 @@ def getSourceArchive() {
     )
 }
 
+def buildDockerEnv(String dockerfile = 'Dockerfile', String extraArgs = '') {
+    def buildEnv
+    docker.withRegistry('https://ghcr.io', 'github-packages-token') {
+        buildEnv = docker.build("ci/realm-core:${dockerfile}", ". -f ${dockerfile} ${extraArgs}")
+    }
+    return buildEnv
+}
+
 enum TestAction {
     None(0x0),
     Build(0x1),

android.Dockerfile

@@ -1,44 +1,2 @@
-FROM ubuntu:20.04
-
-ARG NDK_VERSION=r23
-ARG CMAKE_VERSION=3.21.3
-
-ENV DEBIAN_FRONTEND "noninteractive"
-
-# adb attempts to write under $HOME/.android but if the container is ran as a user other than root
-# as Jenkins does, this will fail
-ENV HOME /tmp
-
-# Locales
-RUN apt-get clean && apt-get update -qq && apt-get install -y locales && locale-gen en_US.UTF-8
-ENV LANG "en_US.UTF-8"
-ENV LANGUAGE "en_US.UTF-8"
-ENV LC_ALL "en_US.UTF-8"
-
-# Keep the packages in alphabetical order to make it easy to avoid duplication
-RUN apt-get update -qq \
-    && apt-get install -y adb \
-                          build-essential \
-                          git \
-                          ninja-build \
-                          unzip \
-                          tar \
-                          wget \
-                          zip \
-    && apt-get clean
-
-# Install CMake
-RUN cd /opt \
-    && wget https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION-linux-x86_64.tar.gz \
-    && tar zxvf cmake-$CMAKE_VERSION-linux-x86_64.tar.gz \
-    && rm -f cmake-$CMAKE_VERSION-linux-x86_64.tar.gz
-
-ENV PATH "/opt/cmake-$CMAKE_VERSION-linux-x86_64/bin:$PATH"
-
-# Install the NDK
-RUN cd /opt \
-    && wget https://dl.google.com/android/repository/android-ndk-$NDK_VERSION-linux.zip \
-    && unzip android-ndk-$NDK_VERSION-linux.zip \
-    && rm -f android-ndk-$NDK_VERSION-linux.zip
-
-ENV ANDROID_NDK "/opt/android-ndk-$NDK_VERSION"
+# https://github.com/realm/ci/tree/master/realm/docker/build-android
+FROM ghcr.io/realm/ci/build-env-android-ndk:master

packaging.Dockerfile

@@ -0,0 +1,2 @@
+# https://github.com/realm/ci/tree/master/realm/docker/build-centos
+FROM ghcr.io/realm/ci/build-env-centos:master