Skip to content

Commit

Permalink
[Geoip] Mmdb hot reload support (envoyproxy#32490)
Browse files Browse the repository at this point in the history 
Commit Message:
Additional Description:
Risk Level:
Testing:
Docs Changes:
Release Notes:
Platform Specific Features: NA
Fixes envoyproxy#30250

---------

Signed-off-by: Kateryna Nezdolii <[email protected]>
  • Loading branch information
@nezdolik
nezdolik authored Aug 22, 2024
1 parent f615e14 commit df382a9
Show file tree
Hide file tree
Showing 15 changed files with 3,069 additions and 104 deletions.
3 changes: 3 additions & 0 deletions changelogs/current.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,5 +227,8 @@ new_features:
:ref:`happy_eyeballs_config <envoy_v3_api_field_config.cluster.v3.UpstreamConnectionOptions.happy_eyeballs_config>`.
A default configuration will be used if not provided. This behavior can be reverted
by setting the runtime guard ``envoy.reloadable_features.use_config_in_happy_eyeballs`` to false.
- area: geoip
change: |
Added ``envoy.reloadable_features.mmdb_files_reload_enabled`` runtime flag that enables reload of mmdb files by default.
deprecated:
2 changes: 2 additions & 0 deletions docs/root/configuration/http/http_filters/geoip_filter.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,5 +76,7 @@ per geolocation database type (rooted at ``<stat_prefix>.maxmind.``). Database t
``<db_type>.total``, Counter, Total number of lookups performed for a given geolocation database file.
``<db_type>.hit``, Counter, Total number of successful lookups (with non empty lookup result) performed for a given geolocation database file.
``<db_type>.lookup_error``, Counter, Total number of errors that occured during lookups for a given geolocation database file.
``<db_type>.db_reload_success``, Counter, Total number of times when the geolocation database file was reloaded successfully.
``<db_type>.db_reload_error``, Counter, Total number of times when the geolocation database file failed to reload.


1 change: 1 addition & 0 deletions source/common/runtime/runtime_features.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ RUNTIME_GUARD(envoy_reloadable_features_internal_authority_header_validator);
RUNTIME_GUARD(envoy_reloadable_features_jwt_authn_remove_jwt_from_query_params);
RUNTIME_GUARD(envoy_reloadable_features_jwt_authn_validate_uri);
RUNTIME_GUARD(envoy_reloadable_features_lua_flow_control_while_http_call);
RUNTIME_GUARD(envoy_reloadable_features_mmdb_files_reload_enabled);
RUNTIME_GUARD(envoy_reloadable_features_no_extension_lookup_by_name);
RUNTIME_GUARD(envoy_reloadable_features_no_timer_based_rate_limit_token_bucket);
RUNTIME_GUARD(envoy_reloadable_features_original_dst_rely_on_idle_timeout);
Expand Down
1 change: 1 addition & 0 deletions source/extensions/geoip_providers/maxmind/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ envoy_cc_library(
deps = [
"//bazel/foreign_cc:maxmind_linux_darwin",
"//envoy/geoip:geoip_provider_driver_interface",
"//source/common/common:thread_synchronizer_lib",
"@envoy_api//envoy/extensions/geoip_providers/maxmind/v3:pkg_cc_proto",
],
)
4 changes: 3 additions & 1 deletion source/extensions/geoip_providers/maxmind/config.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,9 @@ class DriverSingleton : public Envoy::Singleton::Instance {
} else {
const auto& provider_config =
std::make_shared<GeoipProviderConfig>(proto_config, stat_prefix, context.scope());
driver = std::make_shared<GeoipProvider>(singleton, provider_config);
driver = std::make_shared<GeoipProvider>(
context.serverFactoryContext().mainThreadDispatcher(),
context.serverFactoryContext().api(), singleton, provider_config);
drivers_[key] = driver;
}
return driver;
Expand Down
222 changes: 179 additions & 43 deletions source/extensions/geoip_providers/maxmind/geoip_provider.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

#include "source/common/common/assert.h"
#include "source/common/protobuf/protobuf.h"
#include "source/common/runtime/runtime_features.h"

namespace Envoy {
namespace Extensions {
Expand All @@ -16,6 +17,10 @@ static constexpr const char* MMDB_ASN_LOOKUP_ARGS[] = {"autonomous_system_number
static constexpr const char* MMDB_ANON_LOOKUP_ARGS[] = {"is_anonymous", "is_anonymous_vpn",
"is_hosting_provider", "is_tor_exit_node",
"is_public_proxy"};

static constexpr absl::string_view CITY_DB_TYPE = "city_db";
static constexpr absl::string_view ISP_DB_TYPE = "isp_db";
static constexpr absl::string_view ANON_DB_TYPE = "anon_db";
} // namespace

GeoipProviderConfig::GeoipProviderConfig(
Expand Down Expand Up @@ -60,20 +65,22 @@ GeoipProviderConfig::GeoipProviderConfig(
"city_db_path, isp_db_path or anon_db_path");
}
if (city_db_path_) {
registerGeoDbStats("city_db");
registerGeoDbStats(CITY_DB_TYPE);
}
if (isp_db_path_) {
registerGeoDbStats("isp_db");
registerGeoDbStats(ISP_DB_TYPE);
}
if (anon_db_path_) {
registerGeoDbStats("anon_db");
registerGeoDbStats(ANON_DB_TYPE);
}
};

void GeoipProviderConfig::registerGeoDbStats(const std::string& db_type) {
void GeoipProviderConfig::registerGeoDbStats(const absl::string_view& db_type) {
stat_name_set_->rememberBuiltin(absl::StrCat(db_type, ".total"));
stat_name_set_->rememberBuiltin(absl::StrCat(db_type, ".hit"));
stat_name_set_->rememberBuiltin(absl::StrCat(db_type, ".lookup_error"));
stat_name_set_->rememberBuiltin(absl::StrCat(db_type, ".db_reload_error"));
stat_name_set_->rememberBuiltin(absl::StrCat(db_type, ".db_reload_success"));
}

bool GeoipProviderConfig::isLookupEnabledForHeader(const absl::optional<std::string>& header) {
Expand All @@ -84,30 +91,57 @@ void GeoipProviderConfig::incCounter(Stats::StatName name) {
stats_scope_->counterFromStatName(name).inc();
}

GeoipProvider::GeoipProvider(Event::Dispatcher& dispatcher, Api::Api& api,
Singleton::InstanceSharedPtr owner,
GeoipProviderConfigSharedPtr config)
: config_(config), owner_(owner) {
city_db_ =
config_->cityDbPath() ? initMaxmindDb(config_->cityDbPath().value(), CITY_DB_TYPE) : nullptr;
isp_db_ =
config_->ispDbPath() ? initMaxmindDb(config_->ispDbPath().value(), ISP_DB_TYPE) : nullptr;
anon_db_ =
config_->anonDbPath() ? initMaxmindDb(config_->anonDbPath().value(), ANON_DB_TYPE) : nullptr;
mmdb_reload_dispatcher_ = api.allocateDispatcher("mmdb_reload_routine");
mmdb_watcher_ = dispatcher.createFilesystemWatcher();
mmdb_reload_thread_ = api.threadFactory().createThread(
[this]() -> void {
ENVOY_LOG_MISC(debug, "Started mmdb_reload_routine");
if (config_->cityDbPath() &&
Runtime::runtimeFeatureEnabled("envoy.reloadable_features.mmdb_files_reload_enabled")) {
THROW_IF_NOT_OK(mmdb_watcher_->addWatch(
config_->cityDbPath().value(), Filesystem::Watcher::Events::MovedTo,
[this](uint32_t) {
return onMaxmindDbUpdate(config_->cityDbPath().value(), CITY_DB_TYPE);
}));
}
if (config_->ispDbPath() &&
Runtime::runtimeFeatureEnabled("envoy.reloadable_features.mmdb_files_reload_enabled")) {
THROW_IF_NOT_OK(mmdb_watcher_->addWatch(
config_->ispDbPath().value(), Filesystem::Watcher::Events::MovedTo, [this](uint32_t) {
return onMaxmindDbUpdate(config_->ispDbPath().value(), ISP_DB_TYPE);
}));
}
if (config_->anonDbPath() &&
Runtime::runtimeFeatureEnabled("envoy.reloadable_features.mmdb_files_reload_enabled")) {
THROW_IF_NOT_OK(mmdb_watcher_->addWatch(
config_->anonDbPath().value(), Filesystem::Watcher::Events::MovedTo,
[this](uint32_t) {
return onMaxmindDbUpdate(config_->anonDbPath().value(), ANON_DB_TYPE);
}));
}
mmdb_reload_dispatcher_->run(Event::Dispatcher::RunType::RunUntilExit);
},
Thread::Options{std::string("mmdb_reload_routine")});
};

GeoipProvider::~GeoipProvider() {
ENVOY_LOG(debug, "Shutting down Maxmind geolocation provider");
if (city_db_) {
MMDB_close(city_db_.get());
}
if (isp_db_) {
MMDB_close(isp_db_.get());
if (mmdb_reload_dispatcher_) {
mmdb_reload_dispatcher_->exit();
}
if (anon_db_) {
MMDB_close(anon_db_.get());
}
}

MaxmindDbPtr GeoipProvider::initMaxMindDb(const absl::optional<std::string>& db_path) {
if (db_path) {
MMDB_s maxmind_db;
int result_code = MMDB_open(db_path.value().c_str(), MMDB_MODE_MMAP, &maxmind_db);
RELEASE_ASSERT(MMDB_SUCCESS == result_code,
fmt::format("Unable to open Maxmind database file {}. Error {}", db_path.value(),
std::string(MMDB_strerror(result_code))));
return std::make_unique<MMDB_s>(maxmind_db);
} else {
ENVOY_LOG(debug, "Geolocation database path is empty, skipping database creation");
return nullptr;
if (mmdb_reload_thread_) {
mmdb_reload_thread_->join();
mmdb_reload_thread_.reset();
}
}

Expand All @@ -127,11 +161,19 @@ void GeoipProvider::lookupInCityDb(
if (config_->isLookupEnabledForHeader(config_->cityHeader()) ||
config_->isLookupEnabledForHeader(config_->regionHeader()) ||
config_->isLookupEnabledForHeader(config_->countryHeader())) {
ASSERT(city_db_, "Maxmind city database is not initialised for performing lookups");
int mmdb_error;
const uint32_t n_prev_hits = lookup_result.size();
auto city_db_ptr = getCityDb();
// Used for testing.
synchronizer_.syncPoint(std::string(CITY_DB_TYPE).append("_lookup_pre_complete"));
if (!city_db_ptr) {
IS_ENVOY_BUG("Maxmind city database must be initialised for performing lookups");
return;
}
auto city_db = city_db_ptr.get();
MMDB_lookup_result_s mmdb_lookup_result = MMDB_lookup_sockaddr(
city_db_.get(), reinterpret_cast<const sockaddr*>(remote_address->sockAddr()), &mmdb_error);
city_db->mmdb(), reinterpret_cast<const sockaddr*>(remote_address->sockAddr()),
&mmdb_error);
const uint32_t n_prev_hits = lookup_result.size();
if (!mmdb_error) {
MMDB_entry_data_list_s* entry_data_list;
int status = MMDB_get_entry_data_list(&mmdb_lookup_result.entry, &entry_data_list);
Expand All @@ -152,27 +194,34 @@ void GeoipProvider::lookupInCityDb(
MMDB_COUNTRY_LOOKUP_ARGS[1]);
}
if (lookup_result.size() > n_prev_hits) {
config_->incHit("city_db");
config_->incHit(CITY_DB_TYPE);
}
MMDB_free_entry_data_list(entry_data_list);
}

} else {
config_->incLookupError("city_db");
config_->incLookupError(CITY_DB_TYPE);
}
config_->incTotal("city_db");
config_->incTotal(CITY_DB_TYPE);
}
}

void GeoipProvider::lookupInAsnDb(
const Network::Address::InstanceConstSharedPtr& remote_address,
absl::flat_hash_map<std::string, std::string>& lookup_result) const {
if (config_->isLookupEnabledForHeader(config_->asnHeader())) {
RELEASE_ASSERT(isp_db_, "Maxmind asn database is not initialized for performing lookups");
int mmdb_error;
const uint32_t n_prev_hits = lookup_result.size();
auto isp_db_ptr = getIspDb();
// Used for testing.
synchronizer_.syncPoint(std::string(ISP_DB_TYPE).append("_lookup_pre_complete"));
if (!isp_db_ptr) {
IS_ENVOY_BUG("Maxmind asn database must be initialised for performing lookups");
return;
}
MMDB_lookup_result_s mmdb_lookup_result = MMDB_lookup_sockaddr(
isp_db_.get(), reinterpret_cast<const sockaddr*>(remote_address->sockAddr()), &mmdb_error);
isp_db_ptr->mmdb(), reinterpret_cast<const sockaddr*>(remote_address->sockAddr()),
&mmdb_error);
const uint32_t n_prev_hits = lookup_result.size();
if (!mmdb_error) {
MMDB_entry_data_list_s* entry_data_list;
int status = MMDB_get_entry_data_list(&mmdb_lookup_result.entry, &entry_data_list);
Expand All @@ -181,25 +230,33 @@ void GeoipProvider::lookupInAsnDb(
MMDB_ASN_LOOKUP_ARGS[0]);
MMDB_free_entry_data_list(entry_data_list);
if (lookup_result.size() > n_prev_hits) {
config_->incHit("isp_db");
config_->incHit(ISP_DB_TYPE);
}
} else {
config_->incLookupError("isp_db");
config_->incLookupError(ISP_DB_TYPE);
}
}
config_->incTotal("isp_db");
config_->incTotal(ISP_DB_TYPE);
}
}

void GeoipProvider::lookupInAnonDb(
const Network::Address::InstanceConstSharedPtr& remote_address,
absl::flat_hash_map<std::string, std::string>& lookup_result) const {
if (config_->isLookupEnabledForHeader(config_->anonHeader()) || config_->anonVpnHeader()) {
ASSERT(anon_db_, "Maxmind city database is not initialised for performing lookups");
int mmdb_error;
const uint32_t n_prev_hits = lookup_result.size();
auto anon_db_ptr = getAnonDb();
// Used for testing.
synchronizer_.syncPoint(std::string(ANON_DB_TYPE).append("_lookup_pre_complete"));
if (!anon_db_ptr) {
IS_ENVOY_BUG("Maxmind anon database must be initialised for performing lookups");
return;
}
auto anon_db = anon_db_ptr.get();
MMDB_lookup_result_s mmdb_lookup_result = MMDB_lookup_sockaddr(
anon_db_.get(), reinterpret_cast<const sockaddr*>(remote_address->sockAddr()), &mmdb_error);
anon_db->mmdb(), reinterpret_cast<const sockaddr*>(remote_address->sockAddr()),
&mmdb_error);
const uint32_t n_prev_hits = lookup_result.size();
if (!mmdb_error) {
MMDB_entry_data_list_s* entry_data_list;
int status = MMDB_get_entry_data_list(&mmdb_lookup_result.entry, &entry_data_list);
Expand All @@ -225,15 +282,94 @@ void GeoipProvider::lookupInAnonDb(
config_->anonProxyHeader().value(), MMDB_ANON_LOOKUP_ARGS[4]);
}
if (lookup_result.size() > n_prev_hits) {
config_->incHit("anon_db");
config_->incHit(ANON_DB_TYPE);
}
MMDB_free_entry_data_list(entry_data_list);
} else {
config_->incLookupError("anon_db");
config_->incLookupError(ANON_DB_TYPE);
}
}
config_->incTotal("anon_db");
config_->incTotal(ANON_DB_TYPE);
}
}

MaxmindDbSharedPtr GeoipProvider::initMaxmindDb(const std::string& db_path,
const absl::string_view& db_type, bool reload) {
MMDB_s maxmind_db;
int result_code = MMDB_open(db_path.c_str(), MMDB_MODE_MMAP, &maxmind_db);

if (reload && MMDB_SUCCESS != result_code) {
ENVOY_LOG(error, "Failed to reload Maxmind database {} from file {}. Error {}", db_type,
db_path, std::string(MMDB_strerror(result_code)));
return nullptr;
} else if (MMDB_SUCCESS != result_code) {
// Crash if this is a failure during initial load.
RELEASE_ASSERT(MMDB_SUCCESS == result_code,
fmt::format("Unable to open Maxmind database file {}. Error {}", db_path,
std::string(MMDB_strerror(result_code))));
return nullptr;
}

ENVOY_LOG(info, "Succeeded to reload Maxmind database {} from file {}.", db_type, db_path);
return std::make_shared<MaxmindDb>(std::move(maxmind_db));
}

absl::Status GeoipProvider::mmdbReload(const MaxmindDbSharedPtr reloaded_db,
const absl::string_view& db_type) {
if (reloaded_db) {
if (db_type == CITY_DB_TYPE) {
updateCityDb(reloaded_db);
config_->incDbReloadSuccess(db_type);
} else if (db_type == ISP_DB_TYPE) {
updateIspDb(reloaded_db);
config_->incDbReloadSuccess(db_type);
} else if (db_type == ANON_DB_TYPE) {
updateAnonDb(reloaded_db);
config_->incDbReloadSuccess(db_type);
} else {
ENVOY_LOG(error, "Unsupported maxmind db type {}", db_type);
return absl::InvalidArgumentError(fmt::format("Unsupported maxmind db type {}", db_type));
}
} else {
config_->incDbReloadError(db_type);
}
return absl::OkStatus();
}

MaxmindDbSharedPtr GeoipProvider::getCityDb() const ABSL_LOCKS_EXCLUDED(mmdb_mutex_) {
absl::ReaderMutexLock lock(&mmdb_mutex_);
return city_db_;
}

void GeoipProvider::updateCityDb(MaxmindDbSharedPtr city_db) ABSL_LOCKS_EXCLUDED(mmdb_mutex_) {
absl::MutexLock lock(&mmdb_mutex_);
city_db_ = city_db;
}

MaxmindDbSharedPtr GeoipProvider::getIspDb() const ABSL_LOCKS_EXCLUDED(mmdb_mutex_) {
absl::ReaderMutexLock lock(&mmdb_mutex_);
return isp_db_;
}

void GeoipProvider::updateIspDb(MaxmindDbSharedPtr isp_db) ABSL_LOCKS_EXCLUDED(mmdb_mutex_) {
absl::MutexLock lock(&mmdb_mutex_);
isp_db_ = isp_db;
}

MaxmindDbSharedPtr GeoipProvider::getAnonDb() const ABSL_LOCKS_EXCLUDED(mmdb_mutex_) {
absl::ReaderMutexLock lock(&mmdb_mutex_);
return anon_db_;
}

void GeoipProvider::updateAnonDb(MaxmindDbSharedPtr anon_db) ABSL_LOCKS_EXCLUDED(mmdb_mutex_) {
absl::MutexLock lock(&mmdb_mutex_);
anon_db_ = anon_db;
}

absl::Status GeoipProvider::onMaxmindDbUpdate(const std::string& db_path,
const absl::string_view& db_type) {
MaxmindDbSharedPtr reloaded_db = initMaxmindDb(db_path, db_type, true /* reload */);
return mmdbReload(reloaded_db, db_type);
}

template <class... Params>
Expand Down
Loading

0 comments on commit df382a9

Please sign in to comment.