Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Wrap more libcurl calls #2421

Merged
merged 3 commits into from
Apr 5, 2023
Merged

Wrap more libcurl calls #2421

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
19b872d
Wrap more libcurl calls
Hind-M Mar 29, 2023
d6b2a12
Code review
Hind-M Apr 4, 2023
9dacf46
Move back curl_initialized to context
Hind-M Apr 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion libmamba/include/mamba/core/url.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,6 @@ namespace mamba
std::string decode_url(const std::string& url);
// Only returns a cache name without extension
std::string cache_name_from_url(const std::string& url);
std::string hide_secrets(const std::string_view& str);

class URLHandler
{
Expand Down
2 changes: 2 additions & 0 deletions libmamba/include/mamba/core/util.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -331,6 +331,8 @@ namespace mamba

std::optional<std::string> proxy_match(const std::string& url);

std::string hide_secrets(std::string_view str);

class non_copyable_base
{
public:
Expand Down
102 changes: 102 additions & 0 deletions libmamba/src/core/curl.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,108 @@
//
// The full license is in the file LICENSE, distributed with this software.

// TODO remove all these includes later?
#include <spdlog/spdlog.h>

#include "mamba/core/mamba_fs.hpp" // for fs::exists
#include "mamba/core/util.hpp" // for hide_secrets

#include "curl.hpp"

namespace mamba
{
namespace curl
{
void configure_curl_handle(
CURL* handle,
const std::string& url,
const bool set_low_speed_opt,
const long connect_timeout_secs,
const bool ssl_no_revoke,
const std::optional<std::string>& proxy,
const std::string& ssl_verify
)
{
curl_easy_setopt(handle, CURLOPT_URL, url.c_str());
curl_easy_setopt(handle, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1L);

// This can improve throughput significantly, see
// https://github.com/curl/curl/issues/9601
curl_easy_setopt(handle, CURLOPT_BUFFERSIZE, 100 * 1024);

// DO NOT SET TIMEOUT as it will also take into account multi-start time and
// it's just wrong curl_easy_setopt(m_handle, CURLOPT_TIMEOUT,
// Context::instance().read_timeout_secs);

// TODO while libcurl in conda now _has_ http2 support we need to fix mamba to
// work properly with it this includes:
// - setting the cache stuff correctly
// - fixing how the progress bar works
curl_easy_setopt(handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);

if (set_low_speed_opt)
{
curl_easy_setopt(handle, CURLOPT_LOW_SPEED_TIME, 60L);
curl_easy_setopt(handle, CURLOPT_LOW_SPEED_LIMIT, 30L);
}

curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, connect_timeout_secs);

if (ssl_no_revoke)
{
curl_easy_setopt(handle, CURLOPT_SSL_OPTIONS, CURLSSLOPT_NO_REVOKE);
}

if (proxy)
{
curl_easy_setopt(handle, CURLOPT_PROXY, proxy->c_str());
// TODO LOG_INFO was used here instead; to be modified later following the new log
// procedure (TBD)
spdlog::info("Using Proxy {}", hide_secrets(*proxy));
}

if (ssl_verify.size())
{
if (ssl_verify == "<false>")
{
curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, 0L);
curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST, 0L);
if (proxy)
{
curl_easy_setopt(handle, CURLOPT_PROXY_SSL_VERIFYPEER, 0L);
curl_easy_setopt(handle, CURLOPT_PROXY_SSL_VERIFYHOST, 0L);
}
}
else if (ssl_verify == "<system>")
{
#ifdef LIBMAMBA_STATIC_DEPS
curl_easy_setopt(handle, CURLOPT_CAINFO, nullptr);
if (proxy)
{
curl_easy_setopt(handle, CURLOPT_PROXY_CAINFO, nullptr);
}
#endif
}
else
{
if (!fs::exists(ssl_verify))
{
throw std::runtime_error("ssl_verify does not contain a valid file path.");
}
else
{
curl_easy_setopt(handle, CURLOPT_CAINFO, ssl_verify.c_str());
if (proxy)
{
curl_easy_setopt(handle, CURLOPT_PROXY_CAINFO, ssl_verify.c_str());
}
}
}
}
}
}

/**************
* curl_error *
**************/
Expand Down Expand Up @@ -205,4 +301,10 @@ namespace mamba
set_opt(CURLOPT_HTTPHEADER, p_headers);
return *this;
}

const char* CURLHandle::get_error_buffer() const
{
return m_errorbuffer;
}

} // namespace mamba
18 changes: 16 additions & 2 deletions libmamba/src/core/curl.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#ifndef MAMBA_CURL_HPP
#define MAMBA_CURL_HPP

#include <optional>
#include <stdexcept>
#include <string_view>
#include <utility>
Expand All @@ -22,6 +23,19 @@ extern "C"

namespace mamba
{
namespace curl
{
void configure_curl_handle(
CURL* handle,
const std::string& url,
const bool set_low_speed_opt,
const long connect_timeout_secs,
const bool ssl_no_revoke,
const std::optional<std::string>& proxy,
const std::string& ssl_verify
);
}

enum class CurlLogLevel
{
kInfo,
Expand Down Expand Up @@ -70,13 +84,13 @@ namespace mamba

CURLHandle& set_opt_header();

// TODO Make this private after more wrapping...
char m_errorbuffer[CURL_ERROR_SIZE];
const char* get_error_buffer() const;

private:

CURL* m_handle;
curl_slist* p_headers = nullptr;
char m_errorbuffer[CURL_ERROR_SIZE];
};

template <class T>
Expand Down
91 changes: 13 additions & 78 deletions libmamba/src/core/fetch.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,90 +47,25 @@ namespace mamba

void DownloadTarget::init_curl_handle(CURL* handle, const std::string& url)
{
curl_easy_setopt(handle, CURLOPT_URL, url.c_str());
curl_easy_setopt(handle, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1L);

// This can improve throughput significantly, see
// https://github.com/curl/curl/issues/9601
curl_easy_setopt(handle, CURLOPT_BUFFERSIZE, 100 * 1024);

// DO NOT SET TIMEOUT as it will also take into account multi-start time and
// it's just wrong curl_easy_setopt(m_handle, CURLOPT_TIMEOUT,
// Context::instance().read_timeout_secs);

// TODO while libcurl in conda now _has_ http2 support we need to fix mamba to
// work properly with it this includes:
// - setting the cache stuff correctly
// - fixing how the progress bar works
curl_easy_setopt(handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);

// if the request is slower than 30b/s for 60 seconds, cancel.
std::string no_low_speed_limit = std::getenv("MAMBA_NO_LOW_SPEED_LIMIT")
? std::getenv("MAMBA_NO_LOW_SPEED_LIMIT")
: "0";
if (no_low_speed_limit == "0")
{
curl_easy_setopt(handle, CURLOPT_LOW_SPEED_TIME, 60L);
curl_easy_setopt(handle, CURLOPT_LOW_SPEED_LIMIT, 30L);
}

curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, Context::instance().connect_timeout_secs);

std::string ssl_no_revoke_env = std::getenv("MAMBA_SSL_NO_REVOKE")
? std::getenv("MAMBA_SSL_NO_REVOKE")
: "0";
if (Context::instance().ssl_no_revoke || ssl_no_revoke_env != "0")
{
curl_easy_setopt(handle, CURLOPT_SSL_OPTIONS, CURLSSLOPT_NO_REVOKE);
}

std::optional<std::string> proxy = proxy_match(url);
if (proxy)
{
curl_easy_setopt(handle, CURLOPT_PROXY, proxy->c_str());
LOG_INFO << "Using Proxy " << hide_secrets(*proxy);
}

std::string& ssl_verify = Context::instance().ssl_verify;
if (ssl_verify.size())
{
if (ssl_verify == "<false>")
{
curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, 0L);
curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST, 0L);
if (proxy)
{
curl_easy_setopt(handle, CURLOPT_PROXY_SSL_VERIFYPEER, 0L);
curl_easy_setopt(handle, CURLOPT_PROXY_SSL_VERIFYHOST, 0L);
}
}
else if (ssl_verify == "<system>")
{
#ifdef LIBMAMBA_STATIC_DEPS
curl_easy_setopt(handle, CURLOPT_CAINFO, nullptr);
if (proxy)
{
curl_easy_setopt(handle, CURLOPT_PROXY_CAINFO, nullptr);
}
#endif
}
else
{
if (!fs::exists(ssl_verify))
{
throw std::runtime_error("ssl_verify does not contain a valid file path.");
}
else
{
curl_easy_setopt(handle, CURLOPT_CAINFO, ssl_verify.c_str());
if (proxy)
{
curl_easy_setopt(handle, CURLOPT_PROXY_CAINFO, ssl_verify.c_str());
}
}
}
}
bool set_ssl_no_revoke = (Context::instance().ssl_no_revoke || ssl_no_revoke_env != "0");

curl::configure_curl_handle(
handle,
url,
(no_low_speed_limit == "0"),
Context::instance().connect_timeout_secs,
set_ssl_no_revoke,
proxy_match(url),
Context::instance().ssl_verify
);
}

int
Expand Down Expand Up @@ -628,9 +563,9 @@ namespace mamba
std::stringstream err;
err << "Download error (" << result << ") " << curl_easy_strerror(result) << " ["
<< leffective_url << "]\n";
if (m_curl_handle->m_errorbuffer[0] != '\0')
if (m_curl_handle->get_error_buffer()[0] != '\0')
{
err << m_curl_handle->m_errorbuffer;
err << m_curl_handle->get_error_buffer();
}
LOG_INFO << err.str();

Expand Down
14 changes: 0 additions & 14 deletions libmamba/src/core/url.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,20 +181,6 @@ namespace mamba
return hex_digest.substr(0u, 8u);
}

std::string hide_secrets(const std::string_view& str)
{
std::string copy(str);

if (contains(str, "/t/"))
{
copy = std::regex_replace(copy, Context::instance().token_regex, "/t/*****");
}

copy = std::regex_replace(copy, Context::instance().http_basicauth_regex, "$1$2:*****@");

return copy;
}

URLHandler::URLHandler(const std::string& url)
: m_url(url)
, m_has_scheme(has_scheme(url))
Expand Down
14 changes: 14 additions & 0 deletions libmamba/src/core/util.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1553,4 +1553,18 @@ namespace mamba
return std::nullopt;
}

std::string hide_secrets(std::string_view str)
{
std::string copy(str);

if (contains(str, "/t/"))
{
copy = std::regex_replace(copy, Context::instance().token_regex, "/t/*****");
}

copy = std::regex_replace(copy, Context::instance().http_basicauth_regex, "$1$2:*****@");

return copy;
}

} // namespace mamba