From 19b872d394de9fc33c49a69e5bbb1a7a25dc152a Mon Sep 17 00:00:00 2001 From: Hind Montassif Date: Wed, 29 Mar 2023 14:45:19 +0200 Subject: [PATCH] Wrap more libcurl calls --- libmamba/include/mamba/core/context.hpp | 1 - libmamba/include/mamba/core/fetch.hpp | 2 + libmamba/include/mamba/core/url.hpp | 1 - libmamba/include/mamba/core/util.hpp | 2 + libmamba/src/core/curl.cpp | 102 ++++++++++++++++++++++++ libmamba/src/core/curl.hpp | 18 ++++- libmamba/src/core/fetch.cpp | 98 ++++------------------- libmamba/src/core/url.cpp | 14 ---- libmamba/src/core/util.cpp | 14 ++++ 9 files changed, 153 insertions(+), 99 deletions(-) diff --git a/libmamba/include/mamba/core/context.hpp b/libmamba/include/mamba/core/context.hpp index 0d5ea0e2dc..d73438796a 100644 --- a/libmamba/include/mamba/core/context.hpp +++ b/libmamba/include/mamba/core/context.hpp @@ -190,7 +190,6 @@ namespace mamba bool shell_completion = true; std::string user_agent = "mamba/" LIBMAMBA_VERSION_STRING; - bool curl_initialized = false; int connect_timeout_secs = 10; // int read_timeout_secs = 60; int retry_timeout = 2; // seconds diff --git a/libmamba/include/mamba/core/fetch.hpp b/libmamba/include/mamba/core/fetch.hpp index e618db7c82..71dbbfd6ad 100644 --- a/libmamba/include/mamba/core/fetch.hpp +++ b/libmamba/include/mamba/core/fetch.hpp @@ -129,6 +129,8 @@ namespace mamba std::ofstream m_file; + bool m_curl_initialized; + static std::size_t get_default_retry_timeout(); static void init_curl_handle(CURL* handle, const std::string& url); std::function download_repr(); diff --git a/libmamba/include/mamba/core/url.hpp b/libmamba/include/mamba/core/url.hpp index db284bae32..d45130fdf1 100644 --- a/libmamba/include/mamba/core/url.hpp +++ b/libmamba/include/mamba/core/url.hpp @@ -66,7 +66,6 @@ namespace mamba std::string decode_url(const std::string& url); // Only returns a cache name without extension std::string cache_name_from_url(const std::string& url); - std::string hide_secrets(const std::string_view& str); class URLHandler { diff --git a/libmamba/include/mamba/core/util.hpp b/libmamba/include/mamba/core/util.hpp index ca77784bae..3cb86083b5 100644 --- a/libmamba/include/mamba/core/util.hpp +++ b/libmamba/include/mamba/core/util.hpp @@ -331,6 +331,8 @@ namespace mamba std::optional proxy_match(const std::string& url); + std::string hide_secrets(const std::string_view& str); + class non_copyable_base { public: diff --git a/libmamba/src/core/curl.cpp b/libmamba/src/core/curl.cpp index 604e244fa2..fa5fa8eb1a 100644 --- a/libmamba/src/core/curl.cpp +++ b/libmamba/src/core/curl.cpp @@ -4,12 +4,108 @@ // // The full license is in the file LICENSE, distributed with this software. +// TODO remove all these includes later? #include +#include "mamba/core/mamba_fs.hpp" // for fs::exists +#include "mamba/core/util.hpp" // for hide_secrets + #include "curl.hpp" namespace mamba { + namespace curl + { + void set_curl_handle( + CURL* handle, + const std::string& url, + const bool set_low_speed_opt, + const long& connect_timeout_secs, + const bool ssl_no_revoke, + const std::optional proxy, + const std::string& ssl_verify + ) + { + curl_easy_setopt(handle, CURLOPT_URL, url.c_str()); + curl_easy_setopt(handle, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); + curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1L); + + // This can improve throughput significantly, see + // https://github.com/curl/curl/issues/9601 + curl_easy_setopt(handle, CURLOPT_BUFFERSIZE, 100 * 1024); + + // DO NOT SET TIMEOUT as it will also take into account multi-start time and + // it's just wrong curl_easy_setopt(m_handle, CURLOPT_TIMEOUT, + // Context::instance().read_timeout_secs); + + // TODO while libcurl in conda now _has_ http2 support we need to fix mamba to + // work properly with it this includes: + // - setting the cache stuff correctly + // - fixing how the progress bar works + curl_easy_setopt(handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); + + if (set_low_speed_opt) + { + curl_easy_setopt(handle, CURLOPT_LOW_SPEED_TIME, 60L); + curl_easy_setopt(handle, CURLOPT_LOW_SPEED_LIMIT, 30L); + } + + curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, connect_timeout_secs); + + if (ssl_no_revoke) + { + curl_easy_setopt(handle, CURLOPT_SSL_OPTIONS, CURLSSLOPT_NO_REVOKE); + } + + if (proxy) + { + curl_easy_setopt(handle, CURLOPT_PROXY, proxy->c_str()); + // TODO LOG_INFO was used here instead; to be modified later following the new log + // procedure (TBD) + spdlog::info("Using Proxy {}", hide_secrets(*proxy)); + } + + if (ssl_verify.size()) + { + if (ssl_verify == "") + { + curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, 0L); + curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST, 0L); + if (proxy) + { + curl_easy_setopt(handle, CURLOPT_PROXY_SSL_VERIFYPEER, 0L); + curl_easy_setopt(handle, CURLOPT_PROXY_SSL_VERIFYHOST, 0L); + } + } + else if (ssl_verify == "") + { +#ifdef LIBMAMBA_STATIC_DEPS + curl_easy_setopt(handle, CURLOPT_CAINFO, nullptr); + if (proxy) + { + curl_easy_setopt(handle, CURLOPT_PROXY_CAINFO, nullptr); + } +#endif + } + else + { + if (!fs::exists(ssl_verify)) + { + throw std::runtime_error("ssl_verify does not contain a valid file path."); + } + else + { + curl_easy_setopt(handle, CURLOPT_CAINFO, ssl_verify.c_str()); + if (proxy) + { + curl_easy_setopt(handle, CURLOPT_PROXY_CAINFO, ssl_verify.c_str()); + } + } + } + } + } + } + /************** * curl_error * **************/ @@ -205,4 +301,10 @@ namespace mamba set_opt(CURLOPT_HTTPHEADER, p_headers); return *this; } + + const char* CURLHandle::get_error_buffer() const + { + return m_errorbuffer; + } + } // namespace mamba diff --git a/libmamba/src/core/curl.hpp b/libmamba/src/core/curl.hpp index 0cd982337f..2ba5df5c2c 100644 --- a/libmamba/src/core/curl.hpp +++ b/libmamba/src/core/curl.hpp @@ -7,6 +7,7 @@ #ifndef MAMBA_CURL_HPP #define MAMBA_CURL_HPP +#include #include #include #include @@ -22,6 +23,19 @@ extern "C" namespace mamba { + namespace curl + { + void set_curl_handle( + CURL* handle, + const std::string& url, + const bool set_low_speed_opt, + const long& connect_timeout_secs, + const bool ssl_no_revoke, + const std::optional proxy, + const std::string& ssl_verify + ); + } + enum class CurlLogLevel { kInfo, @@ -70,13 +84,13 @@ namespace mamba CURLHandle& set_opt_header(); - // TODO Make this private after more wrapping... - char m_errorbuffer[CURL_ERROR_SIZE]; + const char* get_error_buffer() const; private: CURL* m_handle; curl_slist* p_headers = nullptr; + char m_errorbuffer[CURL_ERROR_SIZE]; }; template diff --git a/libmamba/src/core/fetch.cpp b/libmamba/src/core/fetch.cpp index b0bee52054..9ffc55a745 100644 --- a/libmamba/src/core/fetch.cpp +++ b/libmamba/src/core/fetch.cpp @@ -30,6 +30,7 @@ namespace mamba : m_name(name) , m_filename(filename) , m_url(unc_url(url)) + , m_curl_initialized(false) { m_curl_handle = std::make_unique(); init_curl_ssl(); @@ -47,90 +48,25 @@ namespace mamba void DownloadTarget::init_curl_handle(CURL* handle, const std::string& url) { - curl_easy_setopt(handle, CURLOPT_URL, url.c_str()); - curl_easy_setopt(handle, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); - curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1L); - - // This can improve throughput significantly, see - // https://github.com/curl/curl/issues/9601 - curl_easy_setopt(handle, CURLOPT_BUFFERSIZE, 100 * 1024); - - // DO NOT SET TIMEOUT as it will also take into account multi-start time and - // it's just wrong curl_easy_setopt(m_handle, CURLOPT_TIMEOUT, - // Context::instance().read_timeout_secs); - - // TODO while libcurl in conda now _has_ http2 support we need to fix mamba to - // work properly with it this includes: - // - setting the cache stuff correctly - // - fixing how the progress bar works - curl_easy_setopt(handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); - // if the request is slower than 30b/s for 60 seconds, cancel. std::string no_low_speed_limit = std::getenv("MAMBA_NO_LOW_SPEED_LIMIT") ? std::getenv("MAMBA_NO_LOW_SPEED_LIMIT") : "0"; - if (no_low_speed_limit == "0") - { - curl_easy_setopt(handle, CURLOPT_LOW_SPEED_TIME, 60L); - curl_easy_setopt(handle, CURLOPT_LOW_SPEED_LIMIT, 30L); - } - - curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, Context::instance().connect_timeout_secs); std::string ssl_no_revoke_env = std::getenv("MAMBA_SSL_NO_REVOKE") ? std::getenv("MAMBA_SSL_NO_REVOKE") : "0"; - if (Context::instance().ssl_no_revoke || ssl_no_revoke_env != "0") - { - curl_easy_setopt(handle, CURLOPT_SSL_OPTIONS, CURLSSLOPT_NO_REVOKE); - } - - std::optional proxy = proxy_match(url); - if (proxy) - { - curl_easy_setopt(handle, CURLOPT_PROXY, proxy->c_str()); - LOG_INFO << "Using Proxy " << hide_secrets(*proxy); - } - - std::string& ssl_verify = Context::instance().ssl_verify; - if (ssl_verify.size()) - { - if (ssl_verify == "") - { - curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, 0L); - curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST, 0L); - if (proxy) - { - curl_easy_setopt(handle, CURLOPT_PROXY_SSL_VERIFYPEER, 0L); - curl_easy_setopt(handle, CURLOPT_PROXY_SSL_VERIFYHOST, 0L); - } - } - else if (ssl_verify == "") - { -#ifdef LIBMAMBA_STATIC_DEPS - curl_easy_setopt(handle, CURLOPT_CAINFO, nullptr); - if (proxy) - { - curl_easy_setopt(handle, CURLOPT_PROXY_CAINFO, nullptr); - } -#endif - } - else - { - if (!fs::exists(ssl_verify)) - { - throw std::runtime_error("ssl_verify does not contain a valid file path."); - } - else - { - curl_easy_setopt(handle, CURLOPT_CAINFO, ssl_verify.c_str()); - if (proxy) - { - curl_easy_setopt(handle, CURLOPT_PROXY_CAINFO, ssl_verify.c_str()); - } - } - } - } + bool set_ssl_no_revoke = (Context::instance().ssl_no_revoke || ssl_no_revoke_env != "0"); + + curl::set_curl_handle( + handle, + url, + (no_low_speed_limit == "0"), + Context::instance().connect_timeout_secs, + set_ssl_no_revoke, + proxy_match(url), + Context::instance().ssl_verify + ); } int @@ -159,12 +95,12 @@ namespace mamba { auto& ctx = Context::instance(); - if (!ctx.curl_initialized) + if (!m_curl_initialized) { if (ctx.ssl_verify == "") { LOG_DEBUG << "'ssl_verify' not activated, skipping cURL SSL init"; - ctx.curl_initialized = true; + m_curl_initialized = true; return; } @@ -223,7 +159,7 @@ namespace mamba } } - ctx.curl_initialized = true; + m_curl_initialized = true; } } @@ -628,9 +564,9 @@ namespace mamba std::stringstream err; err << "Download error (" << result << ") " << curl_easy_strerror(result) << " [" << leffective_url << "]\n"; - if (m_curl_handle->m_errorbuffer[0] != '\0') + if (m_curl_handle->get_error_buffer()[0] != '\0') { - err << m_curl_handle->m_errorbuffer; + err << m_curl_handle->get_error_buffer(); } LOG_INFO << err.str(); diff --git a/libmamba/src/core/url.cpp b/libmamba/src/core/url.cpp index 23f6255c42..2d23ae8cbb 100644 --- a/libmamba/src/core/url.cpp +++ b/libmamba/src/core/url.cpp @@ -181,20 +181,6 @@ namespace mamba return hex_digest.substr(0u, 8u); } - std::string hide_secrets(const std::string_view& str) - { - std::string copy(str); - - if (contains(str, "/t/")) - { - copy = std::regex_replace(copy, Context::instance().token_regex, "/t/*****"); - } - - copy = std::regex_replace(copy, Context::instance().http_basicauth_regex, "$1$2:*****@"); - - return copy; - } - URLHandler::URLHandler(const std::string& url) : m_url(url) , m_has_scheme(has_scheme(url)) diff --git a/libmamba/src/core/util.cpp b/libmamba/src/core/util.cpp index 77c6ab64fa..c8448311a8 100644 --- a/libmamba/src/core/util.cpp +++ b/libmamba/src/core/util.cpp @@ -1553,4 +1553,18 @@ namespace mamba return std::nullopt; } + std::string hide_secrets(const std::string_view& str) + { + std::string copy(str); + + if (contains(str, "/t/")) + { + copy = std::regex_replace(copy, Context::instance().token_regex, "/t/*****"); + } + + copy = std::regex_replace(copy, Context::instance().http_basicauth_regex, "$1$2:*****@"); + + return copy; + } + } // namespace mamba