Use cross-compilers instead of CentOS image (#6559)

* Use cross-compilers instead of CentOS image

* changelog

* fix bad merge

* refactor toolchain files

* clarify useToolchain exception circumstances

CHANGELOG.md

@@ -19,7 +19,7 @@
 -----------
 
 ### Internals
-* None.
+* Added CMake toolchains for cross-targeting Linux x86_64, armv7, and aarch64. (PR [#6559](https://github.com/realm/realm-core/pull/6559))
 
 ----------------------------------------------
 

CMakeLists.txt

@@ -297,8 +297,8 @@ endif()
 # Just use -lz and let Xcode figure it out
 # Emscripten does provide Zlib, but it doesn't work with find_package and is handled specially
 if(NOT APPLE AND NOT EMSCRIPTEN AND NOT TARGET ZLIB::ZLIB)
-    if(WIN32)
-        realm_acquire_dependency(zlib ${DEP_WIN32_ZLIB_VERSION} ZLIB_CMAKE_INCLUDE_FILE)
+    if(WIN32 OR (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND REALM_LINUX_TOOLCHAIN))
+        realm_acquire_dependency(zlib ${DEP_ZLIB_VERSION} ZLIB_CMAKE_INCLUDE_FILE)
         include(${ZLIB_CMAKE_INCLUDE_FILE})
     elseif(ANDROID)
         # On Android FindZLIB chooses the static libz over the dynamic one, but this leads to issues

Jenkinsfile

@@ -105,9 +105,9 @@ jobWrapper {
 
         parallelExecutors = [
             buildLinuxRelease       : doBuildLinux('Release'),
-            checkLinuxDebug         : doCheckInDocker(buildOptions),
+            checkLinuxDebug         : doCheckInDocker(buildOptions + [useToolchain : true]),
             checkLinuxDebugEncrypt  : doCheckInDocker(buildOptions + [useEncryption : true]),
-            checkLinuxRelease_4     : doCheckInDocker(buildOptions + [maxBpNodeSize: 4, buildType : 'Release']),
+            checkLinuxRelease_4     : doCheckInDocker(buildOptions + [maxBpNodeSize: 4, buildType : 'Release', useToolchain : true]),
             checkLinuxDebug_Sync    : doCheckInDocker(buildOptions + [enableSync: true, dumpChangesetTransform: true]),
             checkLinuxDebugNoEncryp : doCheckInDocker(buildOptions + [enableEncryption: false]),
             checkMacOsRelease_Sync  : doBuildMacOs(buildOptions + [buildType: 'Release', enableSync: true]),
@@ -251,7 +251,7 @@ def doCheckInDocker(Map options = [:]) {
         rlmNode('docker') {
             getArchive()
 
-            def buildEnv = buildDockerEnv('testing.Dockerfile')
+            def buildEnv = buildDockerEnv('linux.Dockerfile')
 
             def environment = environment()
             environment << 'UNITTEST_XML=unit-test-report.xml'
@@ -260,6 +260,12 @@ def doCheckInDocker(Map options = [:]) {
                 environment << 'UNITTEST_ENCRYPT_ALL=1'
             }
 
+            // We don't enable this by default, because using a toolchain with its own sysroot
+            // prevents CMake from finding system libraries like curl which we use in sync tests.
+            if (options.useToolchain) {
+                cmakeDefinitions += " -DCMAKE_TOOLCHAIN_FILE=\"${env.WORKSPACE}/tools/cmake/x86_64-linux-gnu.toolchain.cmake\""
+            }
+
             def buildSteps = { String dockerArgs = "" ->
                 buildEnv.inside(dockerArgs) {
                     withEnv(environment) {
@@ -351,7 +357,7 @@ def doCheckSanity(Map options = [:]) {
               "UNITTEST_SUITE_NAME=Linux-${options.buildType}",
               "TSAN_OPTIONS=\"suppressions=${WORKSPACE}/test/tsan.suppress\""
             ]
-            buildDockerEnv('testing.Dockerfile').inside(privileged) {
+            buildDockerEnv('linux.Dockerfile').inside(privileged) {
                 withEnv(environment) {
                     try {
                         dir('build-dir') {
@@ -379,12 +385,12 @@ def doBuildLinux(String buildType) {
         rlmNode('docker') {
             getSourceArchive()
 
-            buildDockerEnv('packaging.Dockerfile').inside {
+            buildDockerEnv('linux.Dockerfile').inside {
                 sh """
                    rm -rf build-dir
                    mkdir build-dir
                    cd build-dir
-                   cmake -DCMAKE_BUILD_TYPE=${buildType} -DREALM_NO_TESTS=1 -DREALM_VERSION="${gitDescribeVersion}" -G Ninja ..
+                   cmake -DCMAKE_BUILD_TYPE=${buildType} -DCMAKE_TOOLCHAIN_FILE=../tools/cmake/x86_64-linux-gnu.toolchain.cmake -DREALM_NO_TESTS=1 -DREALM_VERSION="${gitDescribeVersion}" -G Ninja ..
                    ninja
                    cpack -G TGZ
                 """
@@ -410,7 +416,7 @@ def doBuildLinuxClang(String buildType) {
               'CXX=clang++'
             ]
 
-            buildDockerEnv('testing.Dockerfile').inside {
+            buildDockerEnv('linux.Dockerfile').inside {
                 withEnv(environment) {
                     dir('build-dir') {
                         sh "cmake -D CMAKE_BUILD_TYPE=${buildType} -DREALM_NO_TESTS=1 -DREALM_VERSION=\"${gitDescribeVersion}\" -G Ninja .."
@@ -778,7 +784,7 @@ def doBuildCoverage() {
     rlmNode('docker') {
       getArchive()
 
-      buildDockerEnv('testing.Dockerfile').inside {
+      buildDockerEnv('linux.Dockerfile').inside {
         sh '''
           mkdir build
           cd build

dependencies.list

@@ -1,5 +1,5 @@
 PACKAGE_NAME=realm-core
 VERSION=13.13.0
 OPENSSL_VERSION=3.0.8
-WIN32_ZLIB_VERSION=1.2.13
+ZLIB_VERSION=1.2.13
 MDBREALM_TEST_SERVER_TAG=2023-05-05

tools/cmake/RealmConfig.cmake.in

@@ -24,8 +24,8 @@ find_dependency(Threads)
 # so for an iOS build it'll use the path from the Device plaform, which is an error on Simulator.
 # Just use -lz and let Xcode figure it out
 if(TARGET Realm::Sync AND NOT APPLE AND NOT TARGET ZLIB::ZLIB)
-    if(WIN32)
-        realm_acquire_dependency(zlib @DEP_WIN32_ZLIB_VERSION@ ZLIB_CMAKE_INCLUDE_FILE)
+    if(WIN32 OR (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND REALM_LINUX_TOOLCHAIN))
+        realm_acquire_dependency(zlib @DEP_ZLIB_VERSION@ ZLIB_CMAKE_INCLUDE_FILE)
         include(${ZLIB_CMAKE_INCLUDE_FILE})
     elseif(ANDROID)
         # On Android FindZLIB chooses the static libz over the dynamic one, but this leads to issues

tools/cmake/aarch64-linux-gnu.toolchain.cmake

@@ -0,0 +1,3 @@
+set(_TRIPLET "aarch64-unknown-linux-gnu")
+set(_TOOLCHAIN_MD5 a3a14d87f98904c2558b43bbd310f337)
+include("${CMAKE_CURRENT_LIST_DIR}/linux.toolchain.base.cmake")

tools/cmake/armv7-linux-gnueabihf.toolchain.cmake

@@ -0,0 +1,3 @@
+set(_TRIPLET "armv7-unknown-linux-gnueabihf")
+set(_TOOLCHAIN_MD5 fbf817b1428bb35c93be8e6c15f73d7d)
+include("${CMAKE_CURRENT_LIST_DIR}/linux.toolchain.base.cmake")

tools/cmake/linux.toolchain.base.cmake

@@ -0,0 +1,19 @@
+if(NOT CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux" OR NOT CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "x86_64")
+    message(FATAL_ERROR "This toolchain can only be used on x86_64 Linux.")
+endif()
+
+file(DOWNLOAD https://static.realm.io/toolchains/v3/${_TRIPLET}.tar.zst ${CMAKE_BINARY_DIR}/${_TRIPLET}.tar.zst
+     EXPECTED_HASH MD5=${_TOOLCHAIN_MD5} STATUS _DOWNLOAD_STATUS)
+list(GET _DOWNLOAD_STATUS 0 _DOWNLOAD_STATUS_CODE)
+if(NOT ${_DOWNLOAD_STATUS_CODE} EQUAL 0)
+    message(FATAL_ERROR "Error downloading ${_TRIPLET}.tar.zst: ${_DOWNLOAD_STATUS}")
+endif()
+
+file(ARCHIVE_EXTRACT INPUT ${CMAKE_BINARY_DIR}/${_TRIPLET}.tar.zst DESTINATION ${CMAKE_BINARY_DIR})
+include("${CMAKE_BINARY_DIR}/${_TRIPLET}/toolchain.cmake")
+
+set(CMAKE_EXE_LINKER_FLAGS_INIT "-Xlinker --exclude-libs=libgcc.a,libstdc++.a -static-libgcc -static-libstdc++")
+set(CMAKE_SHARED_LINKER_FLAGS_INIT "-Xlinker --exclude-libs=libgcc.a,libstdc++.a -static-libgcc -static-libstdc++")
+set(CMAKE_MODULE_LINKER_FLAGS_INIT "-Xlinker --exclude-libs=libgcc.a,libstdc++.a -static-libgcc -static-libstdc++")
+
+set(REALM_LINUX_TOOLCHAIN ON)

tools/cmake/x86_64-linux-gnu.toolchain.cmake

@@ -0,0 +1,3 @@
+set(_TRIPLET "x86_64-unknown-linux-gnu")
+set(_TOOLCHAIN_MD5 aa99fc16d85adf1bb2f8b63b83542044)
+include("${CMAKE_CURRENT_LIST_DIR}/linux.toolchain.base.cmake")