Added the ability to select OpenSSL 1.1.1n if desired. (#4045)

* Added ability to switch to OpenSSL 1.1.1n

* Fixed OpenSSL compilation challenges - the CRL stuff works now

* Added instructions on using OpenSSL 1.1.1 to README

* Added vcpkg args to handle openssl variants

.vscode/cspell.json

@@ -16,6 +16,7 @@
     ".github/CODEOWNERS",
     ".gitignore",
     ".vscode/cspell.json",
+    "vcpkg-custom-ports",
     "ci.yml",
     "squid.conf*",
     "eng/common/**/*",
@@ -54,6 +55,7 @@
     "Deserializes",
     "DFETCH",
     "DMSVC",
+    "DVCPKG",
     "docfx",
     "DPAPI",
     "DRUN",

CMakeSettings.json

@@ -20,6 +20,64 @@
           "name": "MSVC_USE_STATIC_CRT",
           "value": "True",
           "type": "BOOL"
+        },
+        {
+          "name": "VCPKG_MANIFEST_MODE",
+          "value": "True",
+          "type": "BOOL"
+        }
+      ]
+    },
+    {
+      "name": "x64-DebugWithTests-OpenSSL111",
+      "generator": "Ninja",
+      "configurationType": "Debug",
+      "inheritEnvironments": [ "msvc_x64_x64" ],
+      "buildRoot": "${projectDir}\\out\\build\\${name}",
+      "installRoot": "${projectDir}\\out\\install\\${name}",
+      "cmakeCommandArgs": "",
+      "buildCommandArgs": "-v",
+      "ctestCommandArgs": "",
+      "variables": [
+        {
+          "name": "VCPKG_TARGET_TRIPLET",
+          "value": "x64-windows-static",
+          "type": "STRING"
+        },
+        {
+          "name": "MSVC_USE_STATIC_CRT",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "VCPKG_MANIFEST_MODE",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "VCPKG_OVERLAY_PORTS",
+          "value": "${projectDir}\\vcpkg-custom-ports",
+          "type": "STRING"
+        },
+        {
+          "name": "INSTALL_GTEST",
+          "value": "False",
+          "type": "BOOL"
+        },
+        {
+          "name": "BUILD_TESTING",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "BUILD_TRANSPORT_CURL",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "BUILD_SAMPLES",
+          "value": "True",
+          "type": "BOOL"
         }
       ]
     },

README.md

@@ -324,6 +324,41 @@ The following SDK library releases are available on [vcpkg](https://github.com/m
 
 > NOTE: In case of getting linker errors when consuming the SDK on Windows, make sure that [vcpkg triplet](https://vcpkg.readthedocs.io/en/latest/users/triplets/) being consumed matches the [CRT link flags](https://docs.microsoft.com/cpp/build/reference/md-mt-ld-use-run-time-library?view=msvc-160) being set for your app or library build. See also `MSVC_USE_STATIC_CRT` build flag.
 
+## OpenSSL Version
+
+Several packages within the Azure SDK for C++ use the OpenSSL library. By default, the Azure SDK will use whatever the most recent version of OpenSSL is within the VCPKG repository.
+
+If you need to use a specific version of OpenSSL, you can use the vcpkg custom ports feature to specify the version of OpenSSL to use. 
+For example, if you want to use OpenSSL 1.1.1, you should create a folder named `vcpkg-custom-ports` next to to your vcpkg.json file.
+
+Navigate to your clone of the vcpkg vcpkg repo and execute "git checkout 3b3bd424827a1f7f4813216f6b32b6c61e386b2e" - this will reset your repo to the last version of OpenSSL 1.1.1
+in vcpkg. Then, copy the contents of the `ports/openssl` folder from the vcpkg repo to the `vcpkg-custom-ports` folder you created earlier:
+
+```sh
+cd <your vcpkg repo>
+git checkout 3b3bd424827a1f7f4813216f6b32b6c61e386b2e
+cd ports
+cp -r openssl <the location of the vcpkg-custom-ports directory listed above>
+```
+
+This will copy the port information for OpenSSL 1.1.1n to your vcpkg-custom-ports directory.
+
+Once that is done, you can install the custom port of OpenSSL 1.1.1n using the vcpkg tool:
+
+```sh
+vcpkg install --overlay-ports=<path to the vcpkg-custom-ports above>
+```
+
+If you are building using CMAKE, you can instruct CMAKE to apply the overlay ports using the following command line switches:
+
+```sh
+vcpkg -DVCPKG_MANIFEST_MODE=ON -DVCPKG_OVERLAY_PORTS=<path to the vcpkg-custom-ports above> -DVCPKG_MANIFEST_DIR=<path to the directory containing the vcpkg.json file>
+```
+
+In addition, if you need to consume OpenSSL from a dynamic linked library/shared object, you can set the VCPKG triplet to reflect that you want to build the library with dynamic 
+entries.Set the VCPKG_you can set the environment variable to `x64-windows-static` or `x64-windows-dynamic` depending on whether you want to use the static or dynamic version of OpenSSL. 
+Similarly you can use the x64-linux-dynamic and x64-linux-static triplet to specify consumption of libraries as a shared object or dynamic.
+
 ## Need help
 
 - For reference documentation visit the [Azure SDK for C++ documentation](https://azure.github.io/azure-sdk-for-cpp).

eng/pipelines/templates/jobs/ci.tests.yml

@@ -76,6 +76,8 @@ jobs:
       value: ""
     - name: CmakeArgs
       value: ""
+    - name: VcpkgArgs
+      value: ""
 # Apply to all services running public pipeline
     - name: AZURE_TEST_MODE
       value: "PLAYBACK"
@@ -142,6 +144,7 @@ jobs:
           ServiceDirectory: ${{ parameters.ServiceDirectory }}
           GenerateArgs: "$(CmakeArgs)"
           BuildArgs: "$(BuildArgs)"
+          VcpkgArgs: "$(VcpkgArgs)"
           Env: "$(CmakeEnvArg)"
 
       - ${{ parameters.PreTestSteps }}
@@ -223,6 +226,7 @@ jobs:
             -OsVMImage '$(OSVmImage)'
             -CmakeEnvArg '$(CmakeEnvArg)'
             -BuildArgs '$(BuildArgs)'
+            -VcpkgArgs '$(VcpkgArgs)'
             -Job '$(Agent.JobName)'
             -BuildReason '$(Build.Reason)'
             -SourceBranch '$(Build.SourceBranch)'

eng/pipelines/templates/jobs/live.tests.yml

@@ -62,6 +62,7 @@ jobs:
     CMOCKA_MESSAGE_OUTPUT: "xml"
     AZURE_ENABLE_STATIC_ANALYSIS: 1
     BuildArgs: ""
+    VcpkgArgs: ""
     WindowsCtestConfig: ""
     CmakeEnvArg: ""
     CmakeArgs: ""
@@ -119,6 +120,7 @@ jobs:
     parameters:
       ServiceDirectory: ${{ parameters.ServiceDirectory }}
       GenerateArgs: $(CmakeArgs)
+      VcpkgArgs: "$(VcpkgArgs)"
       BuildArgs: "$(BuildArgs)"
       Env: "$(CmakeEnvArg)"
 

eng/pipelines/templates/stages/platform-matrix.json

@@ -179,6 +179,10 @@
         },
         "included_release": {
           "CMAKE_BUILD_TYPE": "Release"
+        },
+        "openssl_111n": {
+          "CMAKE_BUILD_TYPE": "Release",
+          "VcpkgArgs": " -DVCPKG_MANIFEST_MODE=ON -DVCPKG_OVERLAY_PORTS=$(Build.SourcesDirectory)/vcpkg-custom-ports -DVCPKG_MANIFEST_DIR=$(Build.SourcesDirectory)"
         }
       }
     }

eng/pipelines/templates/steps/cmake-build.yml

@@ -3,6 +3,7 @@ parameters:
   GenerateArgs: ''
   Build: true
   BuildArgs: ''
+  VcpkgArgs: ''
   ServiceDirectory: ''
 
 
@@ -15,7 +16,7 @@ steps:
     displayName: cmake --version
 
   - script: |
-      ${{ parameters.Env }} cmake ${{ parameters.GenerateArgs }} ..
+      ${{ parameters.Env }} cmake ${{ parameters.VcpkgArgs }} ${{ parameters.GenerateArgs }} ..
     workingDirectory: build
     displayName: cmake generate
     env:

eng/scripts/Get-BinarySizes.ps1

@@ -18,6 +18,9 @@ param(
     [Parameter()]
     [string] $BuildArgs,
 
+    [Parameter()]
+	[string] $VcpkgArgs,
+
     [Parameter()]
     [string] $Job,
 

sdk/core/azure-core/CHANGELOG.md

@@ -10,6 +10,7 @@
 
 ### Other Changes
 
+- Added the ability to consume version 1.1.1n of OpenSSL.
 - Added support for Identity token caching, and for configuring token refresh offset in `BearerTokenAuthenticationPolicy`.
 
 ## 1.8.0-beta.1 (2022-10-06)

sdk/core/azure-core/src/http/curl/curl.cpp

@@ -37,24 +37,39 @@
 #include "curl_session_private.hpp"
 
 #if defined(AZ_PLATFORM_POSIX)
+#include <openssl/opensslv.h>
+#if OPENSSL_VERSION_NUMBER < 0x30000000L
+#define USE_OPENSSL_1
+#else
+#define USE_OPENSSL_3
+#endif // OPENSSL_VERSION_NUMBER < 0x30000000L
+
 #include <openssl/asn1t.h>
 #include <openssl/err.h>
+// For OpenSSL > 3.0, we can use the new API to get the certificate's OCSP URL.
+#if defined(USE_OPENSSL_3)
 #include <openssl/http.h>
+#endif
+#if defined(USE_OPENSSL_1)
+#include <openssl/ocsp.h>
+#endif // defined(USE_OPENSSL_1)
 #include <openssl/safestack.h>
 #include <openssl/ssl.h>
 #include <openssl/x509.h>
 #include <openssl/x509_vfy.h>
 #include <openssl/x509v3.h>
+#endif // AZ_PLATFORM_POSIX
+
+#if defined(AZ_PLATFORM_POSIX)
 #include <poll.h> // for poll()
 #include <sys/socket.h> // for socket shutdown
 #elif defined(AZ_PLATFORM_WINDOWS)
 #include <winsock2.h> // for WSAPoll();
-#endif
+#endif // AZ_PLATFORM_POSIX/AZ_PLATFORM_WINDOWS
 
 #include <algorithm>
 #include <chrono>
 #include <iomanip>
-#include <openssl/ssl.h>
 #include <sstream>
 #include <string>
 #include <thread>
@@ -1383,6 +1398,12 @@ namespace Azure { namespace Core {
     {
       using type = BasicUniqueHandle<BIO, BIO_free_all>;
     };
+#if defined(USE_OPENSSL_1)
+    template <> struct UniqueHandleHelper<OCSP_REQ_CTX>
+    {
+      using type = BasicUniqueHandle<OCSP_REQ_CTX, OCSP_REQ_CTX_free>;
+    };
+#endif // USE_OPENSSL_1
 
     template <> struct UniqueHandleHelper<STACK_OF(X509_CRL)>
     {
@@ -1445,8 +1466,105 @@ namespace Azure { namespace Core {
       Azure::Core::_internal::UniqueHandle<X509_CRL> LoadCrlFromUrl(std::string const& url)
       {
         Log::Write(Logger::Level::Informational, "Load CRL from Url: " + url);
-        auto crl = Azure::Core::_internal::MakeUniqueHandle(
+        Azure::Core::_internal::UniqueHandle<X509_CRL> crl;
+#if defined(USE_OPENSSL_3)
+        crl = Azure::Core::_internal::MakeUniqueHandle(
             X509_CRL_load_http, url.c_str(), nullptr, nullptr, 5);
+#else
+        std::string host, port, path;
+        int use_ssl;
+        {
+          char *host_ptr, *port_ptr, *path_ptr;
+          if (!OCSP_parse_url(url.c_str(), &host_ptr, &port_ptr, &path_ptr, &use_ssl))
+          {
+            Log::Write(Logger::Level::Error, "Failure parsing URL");
+            return nullptr;
+          }
+          host = host_ptr;
+          port = port_ptr;
+          path = path_ptr;
+        }
+
+        if (use_ssl)
+        {
+          Log::Write(Logger::Level::Error, "CRL HTTPS not supported");
+          return nullptr;
+        }
+        Azure::Core::_internal::UniqueHandle<BIO> bio{
+            Azure::Core::_internal::MakeUniqueHandle(BIO_new_connect, host.c_str())};
+        if (!bio)
+        {
+          Log::Write(
+              Logger::Level::Error,
+              "BIO_new_connect failed" + _detail::GetOpenSSLError("Load CRL"));
+          return nullptr;
+        }
+        if (!BIO_set_conn_port(bio.get(), const_cast<char*>(port.c_str())))
+        {
+          Log::Write(
+              Logger::Level::Error,
+              "BIO_set_conn_port failed" + _detail::GetOpenSSLError("Load CRL"));
+          return nullptr;
+        }
+
+        auto requestContext
+            = Azure::Core::_internal::MakeUniqueHandle(OCSP_REQ_CTX_new, bio.get(), 1024 * 1024);
+        if (!requestContext)
+        {
+          Log::Write(
+              Logger::Level::Error,
+              "OCSP_REQ_CTX_new failed" + _detail::GetOpenSSLError("Load CRL"));
+          return nullptr;
+        }
+
+        // By default the OCSP APIs limit the CRL length to 1M, that isn't sufficient
+        // for many web sites, so increase it to 10M.
+        OCSP_set_max_response_length(requestContext.get(), 10 * 1024 * 1024);
+
+        if (!OCSP_REQ_CTX_http(requestContext.get(), "GET", url.c_str()))
+        {
+          Log::Write(
+              Logger::Level::Error,
+              "OCSP_REQ_CTX_http failed" + _detail::GetOpenSSLError("Load CRL"));
+          return nullptr;
+        }
+
+        if (!OCSP_REQ_CTX_add1_header(requestContext.get(), "Host", host.c_str()))
+        {
+          Log::Write(
+              Logger::Level::Error,
+              "OCSP_REQ_add1_header failed" + _detail::GetOpenSSLError("Load CRL"));
+          return nullptr;
+        }
+
+        {
+          X509_CRL* crl_ptr = nullptr;
+          int rv;
+          do
+          {
+            rv = X509_CRL_http_nbio(requestContext.get(), &crl_ptr);
+          } while (rv == -1);
+
+          if (rv != 1)
+          {
+            if (ERR_peek_error() == 0)
+            {
+              Log::Write(
+                  Logger::Level::Error,
+                  "X509_CRL_http_nbio failed, possible because CRL is too long.");
+            }
+            else
+            {
+              Log::Write(
+                  Logger::Level::Error,
+                  "X509_CRL_http_nbio failed" + _detail::GetOpenSSLError("Load CRL"));
+            }
+            return nullptr;
+          }
+          crl.reset(crl_ptr);
+        }
+
+#endif
         if (!crl)
         {
           Log::Write(Logger::Level::Error, _detail::GetOpenSSLError("Load CRL"));
@@ -1692,7 +1810,11 @@ namespace Azure { namespace Core {
        * @brief Retrieve the CRL associated with the provided store context, if available.
        *
        */
+#if defined(USE_OPENSSL_3)
       STACK_OF(X509_CRL) * CrlHttpCallback(const X509_STORE_CTX* context, const X509_NAME*)
+#else
+      STACK_OF(X509_CRL) * CrlHttpCallback(X509_STORE_CTX* context, X509_NAME*)
+#endif
       {
         Azure::Core::_internal::UniqueHandle<X509_CRL> crl;
         STACK_OF(DIST_POINT) * crlDistributionPoint;