Refactor SHA hashing and the OpenSSL dependency (#6308)

* Refactor SHA hashing and the OpenSSL dependency

* move submodules

* Update src/realm/util/sha_crypto.cpp

Co-authored-by: Thomas Goyne <[email protected]>

* code review

* changelog

* account for missing Span constructor

---------

Co-authored-by: Thomas Goyne <[email protected]>

.gitmodules

@@ -1,3 +1,9 @@
 [submodule "external/catch"]
 	path = external/catch
 	url = https://github.com/catchorg/Catch2.git
+[submodule "src/external/sha-1"]
+	path = src/external/sha-1
+	url = https://github.com/clibs/sha1.git
+[submodule "src/external/sha-2"]
+	path = src/external/sha-2
+	url = https://github.com/kalven/sha-2.git

CHANGELOG.md

@@ -17,7 +17,7 @@
 -----------
 
 ### Internals
-* None.
+* The OpenSSL dependency is included only exactly where needed. Where OpenSSL is not available and the target system doesn't provide them, we now bundle public domain implementations of the SHA1 and SHA2 hashes. Common hashing operations are exposed by `util/sha_crypto.hpp` so that call sites don't need to reason about the exact hash provider. ([PR #6308](https://github.com/realm/realm-core/pull/6308))
 
 ----------------------------------------------
 

CMakeLists.txt

@@ -222,7 +222,7 @@ if(REALM_ENABLE_SYNC)
     if(CMAKE_SYSTEM_NAME MATCHES "^Windows|Linux|Android")
         set(REALM_NEEDS_OPENSSL TRUE)
     endif()
-elseif(CMAKE_SYSTEM_NAME MATCHES "Linux|Android")
+elseif(REALM_ENABLE_ENCRYPTION AND CMAKE_SYSTEM_NAME MATCHES "Linux|Android")
     set(REALM_NEEDS_OPENSSL TRUE)
 endif()
 

src/realm/CMakeLists.txt

@@ -330,8 +330,24 @@ target_include_directories(Storage INTERFACE
     $<INSTALL_INTERFACE:include>
 )
 
+# On systems without a built-in SHA-1 implementation (or one provided by a dependency)
+# we need to bundle the public domain implementation.
+if(NOT APPLE AND NOT CMAKE_SYSTEM_NAME MATCHES "^Windows" AND NOT REALM_HAVE_OPENSSL)
+    add_library(sha1 OBJECT ../external/sha-1/sha1.c)
+    target_include_directories(Storage PRIVATE ../external/sha-1)
+    target_sources(Storage PRIVATE $<TARGET_OBJECTS:sha1>)
+endif()
+
+# On systems without a built-in SHA-2 implementation (or one provided by a dependency)
+# we need to bundle the public domain implementation.
+# Note: This is also used on Windows because Windows lacks a native SHA224 hash needed for realm encryption
+if(NOT APPLE AND NOT REALM_HAVE_OPENSSL OR WIN32)
+    add_library(sha2 OBJECT ../external/sha-2/sha224.cpp ../external/sha-2/sha256.cpp)
+    target_include_directories(Storage PRIVATE ../external/sha-2)
+    target_sources(Storage PRIVATE $<TARGET_OBJECTS:sha2>)
+endif()
+
 if(CMAKE_SYSTEM_NAME MATCHES "^Windows")
-    target_sources(Storage PRIVATE $<TARGET_OBJECTS:sha_win32>)
     if(CMAKE_SYSTEM_NAME STREQUAL "Windows")
         target_link_libraries(Storage INTERFACE Version.lib psapi.lib)
     endif()
@@ -346,7 +362,7 @@ endif()
 
 target_link_libraries(Storage INTERFACE Threads::Threads)
 
-if(UNIX AND NOT APPLE)
+if(REALM_ENABLE_ENCRYPTION AND UNIX AND NOT APPLE AND REALM_HAVE_OPENSSL)
     target_link_libraries(Storage PUBLIC OpenSSL::Crypto)
 endif()
 

src/realm/sync/CMakeLists.txt

@@ -92,7 +92,7 @@ target_link_libraries(Sync PUBLIC Storage)
 
 if(APPLE AND NOT REALM_FORCE_OPENSSL)
     target_link_options(Sync INTERFACE "SHELL:-framework Security")
-else()
+elseif(REALM_HAVE_OPENSSL)
     target_link_libraries(Sync PUBLIC OpenSSL::SSL)
 endif()
 

src/realm/sync/noinst/server/CMakeLists.txt

@@ -31,6 +31,6 @@ target_link_libraries(SyncServer PUBLIC Sync QueryParser)
 
 if(APPLE AND NOT REALM_FORCE_OPENSSL)
     target_sources(SyncServer PRIVATE crypto_server_apple.mm)
-else()
+elseif(REALM_HAVE_OPENSSL)
     target_sources(SyncServer PRIVATE crypto_server_openssl.cpp)
 endif()

src/realm/util/aes_cryptor.hpp

@@ -16,6 +16,7 @@
  *
  **************************************************************************/
 
+#include <array>
 #include <cstddef>
 #include <memory>
 #include <realm/util/features.h>
@@ -78,13 +79,12 @@ class AESCryptor {
     EVP_CIPHER_CTX* m_ctx;
 #endif
 
-    uint8_t m_aesKey[32];
-    uint8_t m_hmacKey[32];
+    std::array<uint8_t, 32> m_aesKey;
+    std::array<uint8_t, 32> m_hmacKey;
     std::vector<iv_table> m_iv_buffer;
     std::unique_ptr<char[]> m_rw_buffer;
     std::unique_ptr<char[]> m_dst_buffer;
 
-    void calc_hmac(const void* src, size_t len, uint8_t* dst, const uint8_t* key) const;
     bool check_hmac(const void* data, size_t len, const uint8_t* hmac) const;
     void crypt(EncryptionMode mode, off_t pos, char* dst, const char* src, const char* stored_iv) noexcept;
     iv_table& get_iv_table(FileDesc fd, off_t data_pos) noexcept;

src/realm/util/encrypted_file_mapping.cpp

@@ -24,6 +24,7 @@
 #include <realm/util/aes_cryptor.hpp>
 #include <realm/util/errno.hpp>
 #include <realm/utilities.hpp>
+#include <realm/util/sha_crypto.hpp>
 #include <realm/util/terminate.hpp>
 
 #include <cstdlib>
@@ -40,10 +41,8 @@
 
 #if defined(_WIN32)
 #include <Windows.h>
-// 224-bit AES-2 from https://github.com/kalven/sha-2 - Public Domain. Native API
-// does not exist for 224 bits (only 128, 256, etc).
-#include <win32/kalven-sha2/sha224.hpp>
 #include <bcrypt.h>
+#pragma comment(lib, "bcrypt.lib")
 #else
 #include <sys/mman.h>
 #include <unistd.h>
@@ -145,8 +144,8 @@ AESCryptor::AESCryptor(const uint8_t* key)
     : m_rw_buffer(new char[block_size])
     , m_dst_buffer(new char[block_size])
 {
-    memcpy(m_aesKey, key, 32);
-    memcpy(m_hmacKey, key + 32, 32);
+    memcpy(m_aesKey.data(), key, 32);
+    memcpy(m_hmacKey.data(), key + 32, 32);
 
 #if REALM_PLATFORM_APPLE
     // A random iv is passed to CCCryptorReset. This iv is *not used* by Realm; we set it manually prior to
@@ -190,7 +189,7 @@ AESCryptor::~AESCryptor() noexcept
 
 void AESCryptor::check_key(const uint8_t* key)
 {
-    if (memcmp(m_aesKey, key, 32) != 0 || memcmp(m_hmacKey, key + 32, 32) != 0)
+    if (memcmp(m_aesKey.data(), key, 32) != 0 || memcmp(m_hmacKey.data(), key + 32, 32) != 0)
         throw DecryptionFailed();
 }
 
@@ -231,8 +230,8 @@ iv_table& AESCryptor::get_iv_table(FileDesc fd, off_t data_pos) noexcept
 
 bool AESCryptor::check_hmac(const void* src, size_t len, const uint8_t* hmac) const
 {
-    uint8_t buffer[224 / 8];
-    calc_hmac(src, len, buffer, m_hmacKey);
+    std::array<uint8_t, 224 / 8> buffer;
+    hmac_sha224(Span(reinterpret_cast<const uint8_t*>(src), len), buffer, m_hmacKey);
 
     // Constant-time memcmp to avoid timing attacks
     uint8_t result = 0;
@@ -355,7 +354,8 @@ void AESCryptor::write(FileDesc fd, off_t pos, const char* src, size_t size) noe
                 ++iv.iv1;
 
             crypt(mode_Encrypt, pos, m_rw_buffer.get(), src, reinterpret_cast<const char*>(&iv.iv1));
-            calc_hmac(m_rw_buffer.get(), block_size, iv.hmac1, m_hmacKey);
+            hmac_sha224(Span(reinterpret_cast<uint8_t*>(m_rw_buffer.get()), block_size),
+                        Span<uint8_t, 28>(iv.hmac1, 28), m_hmacKey);
             // In the extremely unlikely case that both the old and new versions have
             // the same hash we won't know which IV to use, so bump the IV until
             // they're different.
@@ -405,7 +405,7 @@ void AESCryptor::crypt(EncryptionMode mode, off_t pos, char* dst, const char* sr
     }
 
 #else
-    if (!EVP_CipherInit_ex(m_ctx, EVP_aes_256_cbc(), NULL, m_aesKey, iv, mode))
+    if (!EVP_CipherInit_ex(m_ctx, EVP_aes_256_cbc(), NULL, m_aesKey.data(), iv, mode))
         handle_error();
 
     int len;
@@ -422,61 +422,6 @@ void AESCryptor::crypt(EncryptionMode mode, off_t pos, char* dst, const char* sr
 #endif
 }
 
-void AESCryptor::calc_hmac(const void* src, size_t len, uint8_t* dst, const uint8_t* key) const
-{
-#if REALM_PLATFORM_APPLE
-    CCHmac(kCCHmacAlgSHA224, key, 32, src, len, dst);
-#else
-    uint8_t ipad[64];
-    for (size_t i = 0; i < 32; ++i)
-        ipad[i] = key[i] ^ 0x36;
-    memset(ipad + 32, 0x36, 32);
-
-    uint8_t opad[64] = {0};
-    for (size_t i = 0; i < 32; ++i)
-        opad[i] = key[i] ^ 0x5C;
-    memset(opad + 32, 0x5C, 32);
-
-    // Full hmac operation is sha224(opad + sha224(ipad + data))
-#ifdef _WIN32
-    sha224_state s;
-    sha_init(s);
-    sha_process(s, ipad, 64);
-    sha_process(s, static_cast<const uint8_t*>(src), uint32_t(len));
-    sha_done(s, dst);
-
-    sha_init(s);
-    sha_process(s, opad, 64);
-    sha_process(s, dst, 28); // 28 == SHA224_DIGEST_LENGTH
-    sha_done(s, dst);
-#elif OPENSSL_VERSION_NUMBER >= 0x10101000L // 1.1.1
-    std::unique_ptr<EVP_MD_CTX, void (*)(EVP_MD_CTX*)> ctx = {EVP_MD_CTX_new(), EVP_MD_CTX_free};
-    EVP_DigestInit(ctx.get(), EVP_sha224());
-    EVP_DigestUpdate(ctx.get(), ipad, 64);
-    EVP_DigestUpdate(ctx.get(), static_cast<const uint8_t*>(src), len);
-    EVP_DigestFinal(ctx.get(), dst, nullptr);
-
-    ctx = {EVP_MD_CTX_new(), EVP_MD_CTX_free};
-    EVP_DigestInit(ctx.get(), EVP_sha224());
-    EVP_DigestUpdate(ctx.get(), opad, 64);
-    EVP_DigestUpdate(ctx.get(), dst, SHA224_DIGEST_LENGTH);
-    EVP_DigestFinal(ctx.get(), dst, nullptr);
-#else
-    SHA256_CTX ctx;
-    SHA224_Init(&ctx);
-    SHA256_Update(&ctx, ipad, 64);
-    SHA256_Update(&ctx, static_cast<const uint8_t*>(src), len);
-    SHA256_Final(dst, &ctx);
-
-    SHA224_Init(&ctx);
-    SHA256_Update(&ctx, opad, 64);
-    SHA256_Update(&ctx, dst, SHA224_DIGEST_LENGTH);
-    SHA256_Final(dst, &ctx);
-#endif
-
-#endif
-}
-
 EncryptedFileMapping::EncryptedFileMapping(SharedFileInfo& file, size_t file_offset, void* addr, size_t size,
                                            File::AccessMode access)
     : m_file(file)

src/realm/util/sha_crypto.cpp

@@ -27,9 +27,20 @@
 #include <stdio.h>
 #include <bcrypt.h>
 #pragma comment(lib, "bcrypt.lib")
-#else
+#define REALM_USE_BUNDLED_SHA2 1
+#elif REALM_HAVE_OPENSSL
 #include <openssl/sha.h>
 #include <openssl/evp.h>
+#include <openssl/hmac.h>
+#else
+#include <sha1.h>
+#define REALM_USE_BUNDLED_SHA2 1
+#endif
+
+#ifdef REALM_USE_BUNDLED_SHA2
+#include <sha224.hpp>
+#include <sha256.hpp>
+#include <cstring>
 #endif
 
 namespace {
@@ -100,7 +111,7 @@ struct Hash {
     UCHAR hash_object_buffer[512];
     DWORD hash_size;
 };
-#else
+#elif REALM_HAVE_OPENSSL
 void message_digest(const EVP_MD* digest_type, const char* in_buffer, size_t in_buffer_size,
                     unsigned char* out_buffer, unsigned int* output_size)
 {
@@ -126,6 +137,35 @@ void message_digest(const EVP_MD* digest_type, const char* in_buffer, size_t in_
         throw realm::util::runtime_error("EVP_DigestFinal_ex() failed");
 }
 #endif
+
+#ifdef REALM_USE_BUNDLED_SHA2
+using namespace realm::util;
+template <typename ShaState, size_t digest_length>
+void hmac(Span<const uint8_t> in_buffer, Span<uint8_t, digest_length> out_buffer, Span<const uint8_t, 32> key)
+{
+    uint8_t ipad[64];
+    for (size_t i = 0; i < 32; ++i)
+        ipad[i] = key[i] ^ 0x36;
+    memset(ipad + 32, 0x36, 32);
+
+    uint8_t opad[64] = {0};
+    for (size_t i = 0; i < 32; ++i)
+        opad[i] = key[i] ^ 0x5C;
+    memset(opad + 32, 0x5C, 32);
+
+    // Full hmac operation is sha_alg(opad + sha_alg(ipad + data))
+    ShaState s;
+    sha_init(s);
+    sha_process(s, ipad, 64);
+    sha_process(s, in_buffer.data(), std::uint32_t(in_buffer.size()));
+    sha_done(s, out_buffer.data());
+
+    sha_init(s);
+    sha_process(s, opad, 64);
+    sha_process(s, out_buffer.data(), std::uint32_t(digest_length));
+    sha_done(s, out_buffer.data());
+}
+#endif
 } // namespace
 
 namespace realm {
@@ -139,11 +179,13 @@ void sha1(const char* in_buffer, size_t in_buffer_size, unsigned char* out_buffe
     Algorithm alg(BCRYPT_SHA1_ALGORITHM);
     Hash hash(alg, 20);
     hash.get_hash(reinterpret_cast<PUCHAR>(const_cast<char*>(in_buffer)), DWORD(in_buffer_size), out_buffer);
-#else
+#elif REALM_HAVE_OPENSSL
     const EVP_MD* digest_type = EVP_sha1();
     unsigned int output_size;
     message_digest(digest_type, in_buffer, in_buffer_size, out_buffer, &output_size);
     REALM_ASSERT(output_size == 20);
+#else
+    SHA1(reinterpret_cast<char*>(out_buffer), in_buffer, in_buffer_size);
 #endif
 }
 
@@ -155,11 +197,54 @@ void sha256(const char* in_buffer, size_t in_buffer_size, unsigned char* out_buf
     Algorithm alg(BCRYPT_SHA256_ALGORITHM);
     Hash hash(alg, 32);
     hash.get_hash(reinterpret_cast<PUCHAR>(const_cast<char*>(in_buffer)), DWORD(in_buffer_size), out_buffer);
-#else
+#elif REALM_HAVE_OPENSSL
     const EVP_MD* digest_type = EVP_sha256();
     unsigned int output_size;
     message_digest(digest_type, in_buffer, in_buffer_size, out_buffer, &output_size);
     REALM_ASSERT(output_size == 32);
+#else
+    sha256_state s;
+    sha_init(s);
+    sha_process(s, in_buffer, uint32_t(in_buffer_size));
+    sha_done(s, out_buffer);
+#endif
+}
+
+void hmac_sha224(Span<const uint8_t> in_buffer, Span<uint8_t, 28> out_buffer, Span<const uint8_t, 32> key)
+{
+#if REALM_PLATFORM_APPLE
+    static_assert(CC_SHA224_DIGEST_LENGTH == out_buffer.size());
+    CCHmac(kCCHmacAlgSHA224, key.data(), key.size(), in_buffer.data(), in_buffer.size(), out_buffer.data());
+#elif defined(REALM_USE_BUNDLED_SHA2)
+    static_assert(28 == out_buffer.size());
+    hmac<sha224_state>(in_buffer, out_buffer, key);
+#elif REALM_HAVE_OPENSSL
+    static_assert(SHA224_DIGEST_LENGTH == out_buffer.size());
+    unsigned int hashLen;
+    HMAC(EVP_sha224(), key.data(), static_cast<int>(key.size()), in_buffer.data(), in_buffer.size(),
+         out_buffer.data(), &hashLen);
+    REALM_ASSERT_DEBUG(hashLen == out_buffer.size());
+#else
+#error "No SHA224 digest implementation on this platform."
+#endif
+}
+
+void hmac_sha256(Span<const uint8_t> in_buffer, Span<uint8_t, 32> out_buffer, Span<const uint8_t, 32> key)
+{
+#if REALM_PLATFORM_APPLE
+    static_assert(CC_SHA256_DIGEST_LENGTH == out_buffer.size());
+    CCHmac(kCCHmacAlgSHA256, key.data(), key.size(), in_buffer.data(), in_buffer.size(), out_buffer.data());
+#elif defined(REALM_USE_BUNDLED_SHA2)
+    static_assert(32 == out_buffer.size());
+    hmac<sha256_state>(in_buffer, out_buffer, key);
+#elif REALM_HAVE_OPENSSL
+    static_assert(SHA256_DIGEST_LENGTH == out_buffer.size());
+    unsigned int hashLen;
+    HMAC(EVP_sha256(), key.data(), static_cast<int>(key.size()), in_buffer.data(), in_buffer.size(),
+         out_buffer.data(), &hashLen);
+    REALM_ASSERT_DEBUG(hashLen == out_buffer.size());
+#else
+#error "No SHA56 digest implementation on this platform."
 #endif
 }
 

src/realm/util/sha_crypto.hpp

@@ -20,6 +20,7 @@
 #define REALM_SHA_CRYPTO_HPP
 
 #include <cstddef>
+#include <realm/util/span.hpp>
 
 namespace realm {
 namespace util {
@@ -37,6 +38,9 @@ namespace util {
 void sha1(const char* in_buffer, size_t in_buffer_size, unsigned char* out_buffer);
 void sha256(const char* in_buffer, size_t in_buffer_size, unsigned char* out_buffer);
 
+void hmac_sha224(Span<const uint8_t> in_buffer, Span<uint8_t, 28> out_buffer, Span<const uint8_t, 32> key);
+void hmac_sha256(Span<const uint8_t> in_buffer, Span<uint8_t, 32> out_buffer, Span<const uint8_t, 32> key);
+
 } // namespace util
 } // namespace realm