From 5116b3015fbe3d501a1106c71c589bf247e8b7b6 Mon Sep 17 00:00:00 2001 From: Yura Sorokin Date: Mon, 18 Nov 2024 17:53:33 +0100 Subject: [PATCH 01/11] Temporarily partially reverted the fix for PS-9453 The fix for PS-9453 "percona_telemetry causes a long wait on COND_thd_list due to the absence of the root user" (commit 27468f8) partially reverted as a preparation step for cherry-picking Bug #34741098 "component::deinit() will block if calling any registry update APIs" (commit mysql/mysql-server@d39330f). --- sql/server_component/mysql_command_backend.cc | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/sql/server_component/mysql_command_backend.cc b/sql/server_component/mysql_command_backend.cc index d33872eb799e..b962b1c34210 100644 --- a/sql/server_component/mysql_command_backend.cc +++ b/sql/server_component/mysql_command_backend.cc @@ -75,21 +75,6 @@ mysql_state_machine_status cssm_begin_connect(mysql_async_connect *ctx) { my_h_service h_command_consumer_srv = nullptr; MYSQL_SESSION mysql_session = nullptr; - - /* We need to handle the failure in this function. - Setting mcs_extn->session_svc right after session open is not enough - to handle user lookup errors (and following errors as well) - because in case of this function returns error, mysql->extension is - cleaned up immediately by the caller. The caller does not take care of - session_svc, because it is not aware of this structure. - */ - std::shared_ptr mysql_session_close_guard( - &mysql_session, [mcs_extn](MYSQL_SESSION *mysql_session_ptr) { - if (*mysql_session_ptr == nullptr) return; - mcs_extn->session_svc = nullptr; - srv_session_close(*mysql_session_ptr); - }); - if (mcs_extn->mcs_thd == nullptr || mcs_extn->session_svc == nullptr) { /* Avoid possibility of nested txn in the current thd. @@ -254,7 +239,6 @@ mysql_state_machine_status cssm_begin_connect(mysql_async_connect *ctx) { } mysql->client_flag = 0; /* For handshake */ mysql->server_status = SERVER_STATUS_AUTOCOMMIT; - mysql_session = nullptr; // disable delete quard return STATE_MACHINE_DONE; } From d4a8212450f2c372de83953f91ea9ba28f98bb09 Mon Sep 17 00:00:00 2001 From: Samar Pratap Singh Date: Mon, 16 Oct 2023 04:42:45 +0000 Subject: [PATCH 02/11] Bug#34741098 component::deinit() will block if calling any registry update APIs 1. A no_lock version of registry and registry_registration service is implemented which provides the same functionality without taking any lock on the registry. 2. MySQL command service is updated to use either the lock or no_lock version based on the new flag no_lock_registry added in mcs_ext. 3. The no_lock_registry flag is set to true by health monitor query thread before calling MySQL command service APIs (close, connect). Change-Id: I8ebf8f07cffb8ddc4de0f17c48c10cb15be7dad8 --- components/libminchassis/CMakeLists.txt | 1 + components/libminchassis/minimal_chassis.cc | 14 + components/libminchassis/registry.cc | 335 +------------ components/libminchassis/registry_imp.h | 116 +---- components/libminchassis/registry_no_lock.cc | 460 ++++++++++++++++++ .../libminchassis/registry_no_lock_imp.h | 241 +++++++++ .../services/mysql_command_services.h | 3 +- sql/mysqld.cc | 6 + sql/mysqld.h | 1 + sql/server_component/mysql_command_backend.cc | 118 +++-- .../mysql_command_services_imp.cc | 174 ++++--- .../mysql_command_services_imp.h | 1 + 12 files changed, 923 insertions(+), 547 deletions(-) create mode 100644 components/libminchassis/registry_no_lock.cc create mode 100644 components/libminchassis/registry_no_lock_imp.h diff --git a/components/libminchassis/CMakeLists.txt b/components/libminchassis/CMakeLists.txt index 177375b9b2bd..4ecaf666cd6d 100644 --- a/components/libminchassis/CMakeLists.txt +++ b/components/libminchassis/CMakeLists.txt @@ -34,6 +34,7 @@ SET(LIBMINCHASSIS_SOURCES mysql_component.cc mysql_service_implementation.cc registry.cc + registry_no_lock.cc rwlock_scoped_lock.cc ) diff --git a/components/libminchassis/minimal_chassis.cc b/components/libminchassis/minimal_chassis.cc index ab5d77a3d52a..94ac34ddcfbe 100644 --- a/components/libminchassis/minimal_chassis.cc +++ b/components/libminchassis/minimal_chassis.cc @@ -36,6 +36,7 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include "dynamic_loader_scheme_file_imp.h" #include "minimal_chassis_runtime_error_imp.h" #include "registry_imp.h" +#include "registry_no_lock_imp.h" extern SERVICE_TYPE(registry) imp_mysql_minimal_chassis_registry; @@ -58,10 +59,21 @@ BEGIN_SERVICE_IMPLEMENTATION(mysql_minimal_chassis, registry) mysql_registry_imp::acquire, mysql_registry_imp::acquire_related, mysql_registry_imp::release END_SERVICE_IMPLEMENTATION(); +BEGIN_SERVICE_IMPLEMENTATION(mysql_minimal_chassis_no_lock, registry) +mysql_registry_no_lock_imp::acquire, + mysql_registry_no_lock_imp::acquire_related, + mysql_registry_no_lock_imp::release END_SERVICE_IMPLEMENTATION(); + BEGIN_SERVICE_IMPLEMENTATION(mysql_minimal_chassis, registry_registration) mysql_registry_imp::register_service, mysql_registry_imp::unregister, mysql_registry_imp::set_default END_SERVICE_IMPLEMENTATION(); +BEGIN_SERVICE_IMPLEMENTATION(mysql_minimal_chassis_no_lock, + registry_registration) +mysql_registry_no_lock_imp::register_service, + mysql_registry_no_lock_imp::unregister, + mysql_registry_no_lock_imp::set_default END_SERVICE_IMPLEMENTATION(); + BEGIN_SERVICE_IMPLEMENTATION(mysql_minimal_chassis, registry_query) mysql_registry_imp::iterator_create, mysql_registry_imp::iterator_get, mysql_registry_imp::iterator_next, mysql_registry_imp::iterator_is_valid, @@ -110,7 +122,9 @@ mysql_runtime_error_imp::emit END_SERVICE_IMPLEMENTATION(); BEGIN_COMPONENT_PROVIDES(mysql_minimal_chassis) PROVIDES_SERVICE(mysql_minimal_chassis, registry), + PROVIDES_SERVICE(mysql_minimal_chassis_no_lock, registry), PROVIDES_SERVICE(mysql_minimal_chassis, registry_registration), + PROVIDES_SERVICE(mysql_minimal_chassis_no_lock, registry_registration), PROVIDES_SERVICE(mysql_minimal_chassis, registry_query), PROVIDES_SERVICE(mysql_minimal_chassis, registry_metadata_enumerate), PROVIDES_SERVICE(mysql_minimal_chassis, registry_metadata_query), diff --git a/components/libminchassis/registry.cc b/components/libminchassis/registry.cc index cb94928c3796..41d342d96e4d 100644 --- a/components/libminchassis/registry.cc +++ b/components/libminchassis/registry.cc @@ -22,15 +22,8 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include "component_common.h" -#include "mysql_service_implementation.h" #include "registry_imp.h" -#include -#include -#include -#include -#include - // pfs instrumentation headers #include @@ -52,9 +45,6 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ static PSI_rwlock_key key_rwlock_LOCK_registry; -typedef std::map - my_service_registry; - struct my_h_service_iterator_imp { my_service_registry::const_iterator m_it; minimal_chassis::rwlock_scoped_lock m_lock; @@ -86,14 +76,13 @@ void mysql_registry_imp::init() { mysql_rwlock_init(key_rwlock_LOCK_registry, &mysql_registry_imp::LOCK_registry); } + /** De-initializes registry. De-initializes RW lock, all other structures are cleaned up. */ void mysql_registry_imp::deinit() { - mysql_registry_imp::service_registry.clear(); - mysql_registry_imp::interface_mapping.clear(); - + mysql_registry_no_lock_imp::deinit(); mysql_rwlock_destroy(&mysql_registry_imp::LOCK_registry); } @@ -115,238 +104,6 @@ mysql_registry_imp::lock_registry_for_write() { __LINE__); } -/** - Finds a Service Implementation data structure based on the pointer to - interface struct supplied. Assumes caller has at least a read lock on the - Registry. - - @param interface A pointer to the interface structure of the Service - Implementation to look for. - @return A pointer to respective Service Implementation data structure, or - NULL if no such interface pointer is registered within the Registry. -*/ -mysql_service_implementation * -mysql_registry_imp::get_service_implementation_by_interface( - my_h_service interface) { - my_interface_mapping::const_iterator iter = - mysql_registry_imp::interface_mapping.find(interface); - if (iter == mysql_registry_imp::interface_mapping.cend()) { - return nullptr; - } - - return iter->second; -} - -/** - Gets current reference count for a Service Implementation related to the - specified pointer to the interface structure. Assumes caller has at least - a read lock on the Registry. - - @param interface A pointer to the interface structure of the Service - Implementation to get reference count of. - @return A current reference count for specified Service Implementation. - Returns 0 in case there is no such interface or it is not referenced. -*/ -uint64_t mysql_registry_imp::get_service_implementation_reference_count( - my_h_service interface) { - my_interface_mapping::const_iterator iter = - mysql_registry_imp::interface_mapping.find(interface); - if (iter == mysql_registry_imp::interface_mapping.cend()) { - return -1; - } - - return iter->second->get_reference_count(); -} - -/** - Finds and acquires a Service by name. A name of the Service or the Service - Implementation can be specified. In case of the Service name, the default - Service Implementation for Service specified will be returned. Assumes - caller has at least a read lock on the Registry. - - @param service_name Name of Service or Service Implementation to acquire. - @param [out] out_service Pointer to Service handle to set acquired Service. - @return Status of performed operation - @retval false success - @retval true failure -*/ -bool mysql_registry_imp::acquire_nolock(const char *service_name, - my_h_service *out_service) { - try { - if (out_service == nullptr) { - return true; - } - my_service_registry::const_iterator iter; - - iter = mysql_registry_imp::service_registry.find(service_name); - - if (iter == mysql_registry_imp::service_registry.cend()) return true; - - mysql_service_implementation *imp = iter->second; - imp->add_reference(); - *out_service = imp->interface(); - return false; - } catch (...) { - } - return true; -} - -/** - Releases the Service Implementation previously acquired. After the call to - this method the usage of the Service Implementation handle will lead to - unpredicted results. Assumes caller has at least a read lock on the - Registry. - - @param service Service Implementation handle of already acquired Service. - @return Status of performed operation - @retval false success - @retval true failure -*/ -bool mysql_registry_imp::release_nolock(my_h_service service) { - try { - if (service == nullptr) { - return true; - } - - mysql_service_implementation *service_implementation = - mysql_registry_imp::get_service_implementation_by_interface(service); - if (service_implementation == nullptr) { - return true; - } - return service_implementation->release_reference(); - } catch (...) { - } - return true; -} - -/** - Registers a new Service Implementation. If it is the first Service - Implementation for the specified Service then it is made a default one. - Assumes caller has a write lock on the Registry. - - @param service_implementation_name Name of the Service Implementation to - register. - @param ptr Pointer to the Service Implementation structure. - @return Status of performed operation - @retval false success - @retval true failure -*/ -bool mysql_registry_imp::register_service_nolock( - const char *service_implementation_name, my_h_service ptr) { - try { - std::unique_ptr imp = - std::unique_ptr( - new mysql_service_implementation(ptr, service_implementation_name)); - - if (imp->interface() == nullptr) { - return true; - } - - /* Register the implementation name. */ - std::pair addition_result = - mysql_registry_imp::service_registry.emplace(imp->name_c_str(), - imp.get()); - - /* Fail if it was present already. */ - if (!addition_result.second) { - return true; - } else { - try { - /* Register interface in mapping */ - mysql_registry_imp::interface_mapping.emplace(imp->interface(), - imp.get()); - - /* Register the Service Implementation as default for Service name in - case none were registered before. */ - mysql_registry_imp::service_registry.emplace_hint( - addition_result.first, imp->service_name_c_str(), imp.get()); - } catch (...) { - mysql_registry_imp::service_registry.erase(addition_result.first); - /* unique_ptr still has ownership over implementation object, we - don't have to delete it explicitly. */ - return true; - } - } - - /* Pointer is stored in registry, thous we release ownership. */ - imp.release(); - - return false; - } catch (...) { - } - return true; -} - -/** - Removes previously registered Service Implementation from registry. If it is - the default one for specified Service then any one still registered is made - default. If there is no other, the default entry is removed from the - Registry too. Assumes caller has a write lock on the Registry. - - @param service_implementation_name Name of the Service Implementation to - unregister. - @return Status of performed operation - @retval false success - @retval true Failure. May happen when Service is still being referenced. -*/ -bool mysql_registry_imp::unregister_nolock( - const char *service_implementation_name) { - try { - std::unique_ptr imp; - - { - /* Find the implementation and check if it is not being referenced. */ - my_service_registry::iterator imp_iter = - mysql_registry_imp::service_registry.find( - service_implementation_name); - if (imp_iter == mysql_registry_imp::service_registry.end() || - imp_iter->second->get_reference_count() > 0) { - return true; - } - - /* First remove specified implementation, to not include it in search - for new default one. Take ownership on implementation object. */ - imp.reset(imp_iter->second); - mysql_registry_imp::service_registry.erase(imp_iter); - /* After deletion, implementation iterator is not valid, we go out of - scope to prevent it from being reused. */ - } - - /* Remove interface mapping. */ - mysql_registry_imp::interface_mapping.erase( - mysql_registry_imp::interface_mapping.find(imp->interface())); - - /* Look if it is the default implementation. */ - my_service_registry::iterator default_iter = - mysql_registry_imp::service_registry.find(imp->service_name_c_str()); - if (default_iter == mysql_registry_imp::service_registry.end()) { - /* A Service Implementation and no default present. The state is not - consistent. */ - return true; - } - - if (default_iter->second == imp.get()) { - /* Remove the default implementation too. */ - my_service_registry::iterator new_default_iter = - mysql_registry_imp::service_registry.erase(default_iter); - - /* Search for a new default implementation. */ - if (new_default_iter != mysql_registry_imp::service_registry.end() && - !strcmp(imp->service_name_c_str(), - new_default_iter->second->service_name_c_str())) { - /* Set as default implementation. */ - mysql_service_implementation *new_default = new_default_iter->second; - mysql_registry_imp::service_registry.emplace_hint( - new_default_iter, new_default->service_name_c_str(), new_default); - } - } - - return false; - } catch (...) { - } - return true; -} - /** Finds and acquires a Service by name. A name of the Service or the Service Implementation can be specified. In case of the Service name, the default @@ -363,7 +120,7 @@ DEFINE_BOOL_METHOD(mysql_registry_imp::acquire, minimal_chassis::rwlock_scoped_lock lock(&LOCK_registry, false, __FILE__, __LINE__); - return mysql_registry_imp::acquire_nolock(service_name, out_service); + return mysql_registry_no_lock_imp::acquire(service_name, out_service); } /** @@ -382,37 +139,11 @@ DEFINE_BOOL_METHOD(mysql_registry_imp::acquire, DEFINE_BOOL_METHOD(mysql_registry_imp::acquire_related, (const char *service_name, my_h_service service, my_h_service *out_service)) { - try { - minimal_chassis::rwlock_scoped_lock lock(&mysql_registry_imp::LOCK_registry, - false, __FILE__, __LINE__); + minimal_chassis::rwlock_scoped_lock lock(&mysql_registry_imp::LOCK_registry, + false, __FILE__, __LINE__); - mysql_service_implementation *service_implementation = - mysql_registry_imp::get_service_implementation_by_interface(service); - if (service_implementation == nullptr) { - return true; - } - /* Find dot, the component name is right after the dot. */ - const char *component_part = - strchr(service_implementation->name_c_str(), '.'); - if (component_part == nullptr) { - return true; - } - /* Assure given service_name is not fully qualified. */ - if (strchr(service_name, '.') != nullptr) { - return true; - } - my_string service_implementation_name = - my_string(service_name) + component_part; - /* Try to acquire such Service. */ - if (mysql_registry_imp::acquire_nolock(service_implementation_name.c_str(), - out_service)) { - /* service is not found */ - return true; - } - return false; - } catch (...) { - } - return true; + return mysql_registry_no_lock_imp::acquire_related(service_name, service, + out_service); } /** @@ -429,7 +160,7 @@ DEFINE_BOOL_METHOD(mysql_registry_imp::release, (my_h_service service)) { minimal_chassis::rwlock_scoped_lock lock(&mysql_registry_imp::LOCK_registry, false, __FILE__, __LINE__); - return mysql_registry_imp::release_nolock(service); + return mysql_registry_no_lock_imp::release(service); } /** @@ -449,7 +180,7 @@ DEFINE_BOOL_METHOD(mysql_registry_imp::register_service, minimal_chassis::rwlock_scoped_lock lock(&mysql_registry_imp::LOCK_registry, true, __FILE__, __LINE__); - return mysql_registry_imp::register_service_nolock( + return mysql_registry_no_lock_imp::register_service( service_implementation_name, ptr); } @@ -470,7 +201,7 @@ DEFINE_BOOL_METHOD(mysql_registry_imp::unregister, minimal_chassis::rwlock_scoped_lock lock(&mysql_registry_imp::LOCK_registry, true, __FILE__, __LINE__); - return mysql_registry_imp::unregister_nolock(service_implementation_name); + return mysql_registry_no_lock_imp::unregister(service_implementation_name); } /** @@ -484,31 +215,10 @@ DEFINE_BOOL_METHOD(mysql_registry_imp::unregister, */ DEFINE_BOOL_METHOD(mysql_registry_imp::set_default, (const char *service_implementation_name)) { - try { - my_service_registry::const_iterator iter; - - minimal_chassis::rwlock_scoped_lock lock(&mysql_registry_imp::LOCK_registry, - true, __FILE__, __LINE__); - - /* register the implementation name */ - iter = - mysql_registry_imp::service_registry.find(service_implementation_name); - - if (iter == mysql_registry_imp::service_registry.cend()) { - return true; - } - mysql_service_implementation *imp = iter->second; - /* We have to remove and reinsert value as key, the string pointer will - not be valid if we unregister previous default implementation. */ - iter = mysql_registry_imp::service_registry.erase( - mysql_registry_imp::service_registry.find(imp->service_name_c_str())); - mysql_registry_imp::service_registry.emplace_hint( - iter, imp->service_name_c_str(), imp); + minimal_chassis::rwlock_scoped_lock lock(&mysql_registry_imp::LOCK_registry, + true, __FILE__, __LINE__); - return false; - } catch (...) { - } - return true; + return mysql_registry_no_lock_imp::set_default(service_implementation_name); } /** @@ -543,9 +253,10 @@ DEFINE_BOOL_METHOD(mysql_registry_imp::iterator_create, my_service_registry::const_iterator r = (!service_name_pattern || !*service_name_pattern) - ? mysql_registry_imp::service_registry.cbegin() - : mysql_registry_imp::service_registry.find(service_name_pattern); - if (r == mysql_registry_imp::service_registry.cend()) { + ? mysql_registry_no_lock_imp::service_registry.cbegin() + : mysql_registry_no_lock_imp::service_registry.find( + service_name_pattern); + if (r == mysql_registry_no_lock_imp::service_registry.cend()) { return true; } @@ -595,7 +306,7 @@ DEFINE_BOOL_METHOD(mysql_registry_imp::iterator_get, my_service_registry::const_iterator &iter = reinterpret_cast(iterator)->m_it; - if (iter != mysql_registry_imp::service_registry.cend()) { + if (iter != mysql_registry_no_lock_imp::service_registry.cend()) { *out_name = iter->second->name_c_str(); return false; @@ -623,9 +334,9 @@ DEFINE_BOOL_METHOD(mysql_registry_imp::iterator_next, my_service_registry::const_iterator &iter = reinterpret_cast(iterator)->m_it; - if (iter != mysql_registry_imp::service_registry.cend()) { + if (iter != mysql_registry_no_lock_imp::service_registry.cend()) { ++iter; - return iter == mysql_registry_imp::service_registry.cend(); + return iter == mysql_registry_no_lock_imp::service_registry.cend(); } } catch (...) { } @@ -649,7 +360,7 @@ DEFINE_BOOL_METHOD(mysql_registry_imp::iterator_is_valid, my_service_registry::const_iterator &iter = reinterpret_cast(iterator)->m_it; - return iter == mysql_registry_imp::service_registry.cend(); + return iter == mysql_registry_no_lock_imp::service_registry.cend(); } catch (...) { } return true; @@ -661,7 +372,7 @@ DEFINE_BOOL_METHOD(mysql_registry_imp::iterator_is_valid, types, but also static members with different name, so usage of templates is not enough to reuse that part of code. */ #define REGISTRY_IMP mysql_registry_imp -#define REGISTRY mysql_registry_imp::service_registry +#define REGISTRY mysql_registry_no_lock_imp::service_registry #define REGISTRY_TYPE my_service_registry #define LOCK mysql_registry_imp::LOCK_registry #define ITERATOR_TYPE my_h_service_iterator_imp @@ -672,6 +383,4 @@ DEFINE_BOOL_METHOD(mysql_registry_imp::iterator_is_valid, #include "registry_metadata.cc.inc" /* static members for mysql_registry_imp */ -my_service_registry mysql_registry_imp::service_registry; -mysql_registry_imp::my_interface_mapping mysql_registry_imp::interface_mapping; mysql_rwlock_t mysql_registry_imp::LOCK_registry; diff --git a/components/libminchassis/registry_imp.h b/components/libminchassis/registry_imp.h index 66e5b02fc3aa..f7691758aa8a 100644 --- a/components/libminchassis/registry_imp.h +++ b/components/libminchassis/registry_imp.h @@ -24,26 +24,10 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #ifndef MYSQL_SERVER_REGISTRY_H #define MYSQL_SERVER_REGISTRY_H -#include -#include -#include -#include -#include - -#include "c_string_less.h" -#include "mysql_service_implementation.h" +#include "registry_no_lock_imp.h" #include "rwlock_scoped_lock.h" -typedef std::map - my_service_registry; - -class mysql_registry_imp { - typedef std::map - my_interface_mapping; - - /* contain the actual fields definitions */ - static my_service_registry service_registry; - static my_interface_mapping interface_mapping; +class mysql_registry_imp : public mysql_registry_no_lock_imp { static mysql_rwlock_t LOCK_registry; public: @@ -52,6 +36,7 @@ class mysql_registry_imp { should be empty. Shouldn't be called multiple times. */ static void init(); + /** De-initializes registry. De-initializes RW lock, all other structures are cleaned up. @@ -68,77 +53,7 @@ class mysql_registry_imp { */ static minimal_chassis::rwlock_scoped_lock lock_registry_for_write(); - /** - Gets current reference count for a Service Implementation related to the - specified pointer to the interface structure. Assumes caller has at least - a read lock on the Registry. - - @param interface A pointer to the interface structure of the Service - Implementation to get reference count of. - @return A current reference count for specified Service Implementation. - Returns 0 in case there is no such interface or it is not referenced. - */ - static uint64_t get_service_implementation_reference_count( - my_h_service interface); - - /** - Finds and acquires a Service by name. A name of the Service or the Service - Implementation can be specified. In case of the Service name, the default - Service Implementation for Service specified will be returned. Assumes - caller has at least a read lock on the Registry. - - @param service_name Name of Service or Service Implementation to acquire. - @param [out] out_service Pointer to Service handle to set acquired Service. - @return Status of performed operation - @retval false success - @retval true failure - */ - static bool acquire_nolock(const char *service_name, - my_h_service *out_service); - - /** - Releases the Service Implementation previously acquired. After the call to - this method the usage of the Service Implementation handle will lead to - unpredicted results. Assumes caller has at least a read lock on the - Registry. - - @param service Service Implementation handle of already acquired Service. - @return Status of performed operation - @retval false success - @retval true failure - */ - static bool release_nolock(my_h_service service); - - /** - Registers a new Service Implementation. If it is the first Service - Implementation for the specified Service then it is made a default one. - Assumes caller has a write lock on the Registry. - - @param service_implementation_name Name of the Service Implementation to - register. - @param ptr Pointer to the Service Implementation structure. - @return Status of performed operation - @retval false success - @retval true failure - */ - static bool register_service_nolock(const char *service_implementation_name, - my_h_service ptr); - - /** - Removes previously registered Service Implementation from registry. If it is - the default one for specified Service then any one still registered is made - default. If there is no other, the default entry is removed from the - Registry too. Assumes caller has a write lock on the Registry. - - @param service_implementation_name Name of the Service Implementation to - unregister. - @return Status of performed operation - @retval false success - @retval true Failure. May happen when Service is still being referenced. - */ - static bool unregister_nolock(const char *service_implementation_name); - - public: /* Service Implementations */ + /* Service Implementations */ /** Finds and acquires a Service by name. A name of the Service or the Service Implementation can be specified. In case of the Service name, the default @@ -294,28 +209,13 @@ class mysql_registry_imp { (my_h_service_iterator iterator)); /* This includes metadata-related method implementations that are shared - by registry and dynamic_loader, so we don't duplicate the code. Following - defines set up all required symbols. Unfortunately they are not only the - types, but also static members with different name, so usage of templates - is not enough to reuse that part of code. */ + by registry and dynamic_loader, so we don't duplicate the code. Following + defines set up all required symbols. Unfortunately they are not only the + types, but also static members with different name, so usage of templates + is not enough to reuse that part of code. */ #define OBJECT_ITERATOR my_h_service_iterator #define METADATA_ITERATOR my_h_service_metadata_iterator #include "registry_metadata.h.inc" - - private: - /** - Finds a Service Implementation data structure based on the pointer to - interface struct supplied. Assumes caller has at least a read lock on the - Registry. - - @param interface A pointer to the interface structure of the Service - Implementation to look for. - @return A pointer to respective Service Implementation data structure, or - NULL if no such interface pointer is registered within the Registry. - */ - static mysql_service_implementation *get_service_implementation_by_interface( - my_h_service interface); }; - #endif /* MYSQL_SERVER_REGISTRY_H */ diff --git a/components/libminchassis/registry_no_lock.cc b/components/libminchassis/registry_no_lock.cc new file mode 100644 index 000000000000..131e16218221 --- /dev/null +++ b/components/libminchassis/registry_no_lock.cc @@ -0,0 +1,460 @@ +/* Copyright (c) 2023, Oracle and/or its affiliates. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License, version 2.0, +as published by the Free Software Foundation. + +This program is also distributed with certain software (including +but not limited to OpenSSL) that is licensed under separate terms, +as designated in a particular file or component or in included license +documentation. The authors of MySQL hereby grant you an additional +permission to link the program and your derivative works with the +separately licensed software that they have included with MySQL. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License, version 2.0, for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#include "mysql_service_implementation.h" +#include "registry_no_lock_imp.h" + +#include +#include +#include +#include +#include + +typedef std::map + my_service_registry; + +/** + De-initializes registry, other structures. +*/ +void mysql_registry_no_lock_imp::deinit() { + mysql_registry_no_lock_imp::service_registry.clear(); + mysql_registry_no_lock_imp::interface_mapping.clear(); +} + +/** + Finds a Service Implementation data structure based on the pointer to + interface struct supplied. Assumes caller has at least a read lock on the + Registry. + + @param interface A pointer to the interface structure of the Service + Implementation to look for. + @return A pointer to respective Service Implementation data structure, or + NULL if no such interface pointer is registered within the Registry. +*/ +mysql_service_implementation * +mysql_registry_no_lock_imp::get_service_implementation_by_interface( + my_h_service interface) { + my_interface_mapping::const_iterator iter = + mysql_registry_no_lock_imp::interface_mapping.find(interface); + if (iter == mysql_registry_no_lock_imp::interface_mapping.cend()) { + return nullptr; + } + + return iter->second; +} + +/** + Gets current reference count for a Service Implementation related to the + specified pointer to the interface structure. Assumes caller has at least + a read lock on the Registry. + + @param interface A pointer to the interface structure of the Service + Implementation to get reference count of. + @return A current reference count for specified Service Implementation. + Returns 0 in case there is no such interface or it is not referenced. +*/ +uint64_t mysql_registry_no_lock_imp::get_service_implementation_reference_count( + my_h_service interface) { + my_interface_mapping::const_iterator iter = + mysql_registry_no_lock_imp::interface_mapping.find(interface); + if (iter == mysql_registry_no_lock_imp::interface_mapping.cend()) { + return -1; + } + + return iter->second->get_reference_count(); +} + +/** + Finds and acquires a Service by name. A name of the Service or the Service + Implementation can be specified. In case of the Service name, the default + Service Implementation for Service specified will be returned. Assumes + caller has at least a read lock on the Registry. + + @param service_name Name of Service or Service Implementation to acquire. + @param [out] out_service Pointer to Service handle to set acquired Service. + @return Status of performed operation + @retval false success + @retval true failure +*/ +bool mysql_registry_no_lock_imp::acquire_nolock(const char *service_name, + my_h_service *out_service) { + try { + if (out_service == nullptr) { + return true; + } + my_service_registry::const_iterator iter; + + iter = mysql_registry_no_lock_imp::service_registry.find(service_name); + + if (iter == mysql_registry_no_lock_imp::service_registry.cend()) + return true; + + mysql_service_implementation *imp = iter->second; + imp->add_reference(); + *out_service = imp->interface(); + return false; + } catch (...) { + } + return true; +} + +/** + Releases the Service Implementation previously acquired. After the call to + this method the usage of the Service Implementation handle will lead to + unpredicted results. Assumes caller has at least a read lock on the + Registry. + + @param service Service Implementation handle of already acquired Service. + @return Status of performed operation + @retval false success + @retval true failure +*/ +bool mysql_registry_no_lock_imp::release_nolock(my_h_service service) { + try { + if (service == nullptr) { + return true; + } + + mysql_service_implementation *service_implementation = + mysql_registry_no_lock_imp::get_service_implementation_by_interface( + service); + if (service_implementation == nullptr) { + return true; + } + return service_implementation->release_reference(); + } catch (...) { + } + return true; +} + +/** + Registers a new Service Implementation. If it is the first Service + Implementation for the specified Service then it is made a default one. + Assumes caller has a write lock on the Registry. + + @param service_implementation_name Name of the Service Implementation to + register. + @param ptr Pointer to the Service Implementation structure. + @return Status of performed operation + @retval false success + @retval true failure +*/ +bool mysql_registry_no_lock_imp::register_service_nolock( + const char *service_implementation_name, my_h_service ptr) { + try { + std::unique_ptr imp = + std::unique_ptr( + new mysql_service_implementation(ptr, service_implementation_name)); + + if (imp->interface() == nullptr) { + return true; + } + + /* Register the implementation name. */ + std::pair addition_result = + mysql_registry_no_lock_imp::service_registry.emplace(imp->name_c_str(), + imp.get()); + + /* Fail if it was present already. */ + if (!addition_result.second) { + return true; + } else { + try { + /* Register interface in mapping */ + mysql_registry_no_lock_imp::interface_mapping.emplace(imp->interface(), + imp.get()); + + /* Register the Service Implementation as default for Service name in + case none were registered before. */ + mysql_registry_no_lock_imp::service_registry.emplace_hint( + addition_result.first, imp->service_name_c_str(), imp.get()); + } catch (...) { + mysql_registry_no_lock_imp::service_registry.erase( + addition_result.first); + /* unique_ptr still has ownership over implementation object, we + don't have to delete it explicitly. */ + return true; + } + } + + /* Pointer is stored in registry, thous we release ownership. */ + imp.release(); + + return false; + } catch (...) { + } + return true; +} + +/** + Removes previously registered Service Implementation from registry. If it is + the default one for specified Service then any one still registered is made + default. If there is no other, the default entry is removed from the + Registry too. Assumes caller has a write lock on the Registry. + + @param service_implementation_name Name of the Service Implementation to + unregister. + @return Status of performed operation + @retval false success + @retval true Failure. May happen when Service is still being referenced. +*/ +bool mysql_registry_no_lock_imp::unregister_nolock( + const char *service_implementation_name) { + try { + std::unique_ptr imp; + + { + /* Find the implementation and check if it is not being referenced. */ + my_service_registry::iterator imp_iter = + mysql_registry_no_lock_imp::service_registry.find( + service_implementation_name); + if (imp_iter == mysql_registry_no_lock_imp::service_registry.end() || + imp_iter->second->get_reference_count() > 0) { + return true; + } + + /* First remove specified implementation, to not include it in search + for new default one. Take ownership on implementation object. */ + imp.reset(imp_iter->second); + mysql_registry_no_lock_imp::service_registry.erase(imp_iter); + /* After deletion, implementation iterator is not valid, we go out of + scope to prevent it from being reused. */ + } + + /* Remove interface mapping. */ + mysql_registry_no_lock_imp::interface_mapping.erase( + mysql_registry_no_lock_imp::interface_mapping.find(imp->interface())); + + /* Look if it is the default implementation. */ + my_service_registry::iterator default_iter = + mysql_registry_no_lock_imp::service_registry.find( + imp->service_name_c_str()); + if (default_iter == mysql_registry_no_lock_imp::service_registry.end()) { + /* A Service Implementation and no default present. The state is not + consistent. */ + return true; + } + + if (default_iter->second == imp.get()) { + /* Remove the default implementation too. */ + my_service_registry::iterator new_default_iter = + mysql_registry_no_lock_imp::service_registry.erase(default_iter); + + /* Search for a new default implementation. */ + if (new_default_iter != + mysql_registry_no_lock_imp::service_registry.end() && + !strcmp(imp->service_name_c_str(), + new_default_iter->second->service_name_c_str())) { + /* Set as default implementation. */ + mysql_service_implementation *new_default = new_default_iter->second; + mysql_registry_no_lock_imp::service_registry.emplace_hint( + new_default_iter, new_default->service_name_c_str(), new_default); + } + } + + return false; + } catch (...) { + } + return true; +} + +/** + Finds and acquires a Service by name. A name of the Service or the Service + Implementation can be specified. In case of the Service name, the default + Service Implementation for Service specified will be returned. + + This does not take any lock on the registry. It must not be used unless + absolutely necessary. Use the mysql_registry_imp version instead. + + @param service_name Name of Service or Service Implementation to acquire. + @param [out] out_service Pointer to Service handle to set acquired Service. + @return Status of performed operation + @retval false success + @retval true failure +*/ +DEFINE_BOOL_METHOD(mysql_registry_no_lock_imp::acquire, + (const char *service_name, my_h_service *out_service)) { + return mysql_registry_no_lock_imp::acquire_nolock(service_name, out_service); +} + +/** + Finds a Service by name. If there is a Service Implementation with the same + Component part of name as the input Service then the found Service is + returned. + + This does not take any lock on the registry. It must not be used unless + absolutely necessary. Use the mysql_registry_imp version instead. + + @param service_name Name of Service or Service Implementation to acquire. + @param service Service handle already acquired Service Implementation. + @param [out] out_service Pointer to Service Implementation handle to set + acquired Service Implementation. + @return Status of performed operation + @retval false success + @retval true failure +*/ +DEFINE_BOOL_METHOD(mysql_registry_no_lock_imp::acquire_related, + (const char *service_name, my_h_service service, + my_h_service *out_service)) { + try { + mysql_service_implementation *service_implementation = + mysql_registry_no_lock_imp::get_service_implementation_by_interface( + service); + if (service_implementation == nullptr) { + return true; + } + /* Find dot, the component name is right after the dot. */ + const char *component_part = + strchr(service_implementation->name_c_str(), '.'); + if (component_part == nullptr) { + return true; + } + /* Assure given service_name is not fully qualified. */ + if (strchr(service_name, '.') != nullptr) { + return true; + } + my_string service_implementation_name = + my_string(service_name) + component_part; + /* Try to acquire such Service. */ + if (mysql_registry_no_lock_imp::acquire_nolock( + service_implementation_name.c_str(), out_service)) { + /* service is not found */ + return true; + } + return false; + } catch (...) { + } + return true; +} + +/** + Releases the Service Implementation previously acquired. After the call to + this method the usage of the Service Implementation handle will lead to + unpredicted results. + + This does not take any lock on the registry. It must not be used unless + absolutely necessary. Use the mysql_registry_imp version instead. + + @param service Service Implementation handle of already acquired Service. + @return Status of performed operation + @retval false success + @retval true failure +*/ +DEFINE_BOOL_METHOD(mysql_registry_no_lock_imp::release, + (my_h_service service)) { + return mysql_registry_no_lock_imp::release_nolock(service); +} + +/** + Registers a new Service Implementation. If it is the first Service + Implementation for the specified Service then it is made a default one. + + This does not take any lock on the registry. It must not be used unless + absolutely necessary. Use the mysql_registry_imp version instead. + + @param service_implementation_name Name of the Service Implementation to + register. + @param ptr Pointer to the Service Implementation structure. + @return Status of performed operation + @retval false success + @retval true failure +*/ +DEFINE_BOOL_METHOD(mysql_registry_no_lock_imp::register_service, + (const char *service_implementation_name, + my_h_service ptr)) { + return mysql_registry_no_lock_imp::register_service_nolock( + service_implementation_name, ptr); +} + +/** + Removes previously registered Service Implementation from registry. If it is + the default one for specified Service then any one still registered is made + default. If there is no other, the default entry is removed from the + Registry too. + + This does not take any lock on the registry. It must not be used unless + absolutely necessary. Use the mysql_registry_imp version instead. + + @param service_implementation_name Name of the Service Implementation to + unregister. + @return Status of performed operation + @retval false success + @retval true Failure. May happen when Service is still being referenced. +*/ +DEFINE_BOOL_METHOD(mysql_registry_no_lock_imp::unregister, + (const char *service_implementation_name)) { + return mysql_registry_no_lock_imp::unregister_nolock( + service_implementation_name); +} + +/** + Sets new default Service Implementation for corresponding Service name. + + This does not take any lock on the registry. It must not be used unless + absolutely necessary. Use the mysql_registry_imp version instead. + + @param service_implementation_name Name of the Service Implementation to + set as default one. + @return Status of performed operation + @retval false success + @retval true failure +*/ +DEFINE_BOOL_METHOD(mysql_registry_no_lock_imp::set_default, + (const char *service_implementation_name)) { + try { + my_service_registry::const_iterator iter; + + /* register the implementation name */ + iter = mysql_registry_no_lock_imp::service_registry.find( + service_implementation_name); + + if (iter == mysql_registry_no_lock_imp::service_registry.cend()) { + return true; + } + mysql_service_implementation *imp = iter->second; + /* We have to remove and reinsert value as key, the string pointer will + not be valid if we unregister previous default implementation. */ + iter = mysql_registry_no_lock_imp::service_registry.erase( + mysql_registry_no_lock_imp::service_registry.find( + imp->service_name_c_str())); + mysql_registry_no_lock_imp::service_registry.emplace_hint( + iter, imp->service_name_c_str(), imp); + + return false; + } catch (...) { + } + return true; +} + +/* This includes metadata-related method implementations that are shared + by registry and dynamic_loader, so we don't duplicate the code. Following + defines set up all required symbols. Unfortunately they are not only the + types, but also static members with different name, so usage of templates + is not enough to reuse that part of code. */ +#define REGISTRY_IMP mysql_registry_no_lock_imp +#define REGISTRY mysql_registry_no_lock_imp::service_registry +#define REGISTRY_TYPE my_service_registry + +/* static members for mysql_registry_no_lock_imp */ +my_service_registry mysql_registry_no_lock_imp::service_registry; +mysql_registry_no_lock_imp::my_interface_mapping + mysql_registry_no_lock_imp::interface_mapping; diff --git a/components/libminchassis/registry_no_lock_imp.h b/components/libminchassis/registry_no_lock_imp.h new file mode 100644 index 000000000000..d8ae1225e5db --- /dev/null +++ b/components/libminchassis/registry_no_lock_imp.h @@ -0,0 +1,241 @@ +/* Copyright (c) 2023, Oracle and/or its affiliates. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License, version 2.0, +as published by the Free Software Foundation. + +This program is also distributed with certain software (including +but not limited to OpenSSL) that is licensed under separate terms, +as designated in a particular file or component or in included license +documentation. The authors of MySQL hereby grant you an additional +permission to link the program and your derivative works with the +separately licensed software that they have included with MySQL. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License, version 2.0, for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#ifndef MYSQL_SERVER_REGISTRY_NO_LOCK_H +#define MYSQL_SERVER_REGISTRY_NO_LOCK_H + +#include +#include +#include +#include +#include + +#include "c_string_less.h" +#include "mysql_service_implementation.h" + +typedef std::map + my_service_registry; + +class mysql_registry_no_lock_imp { + protected: + typedef std::map + my_interface_mapping; + + /* contain the actual fields definitions */ + static my_service_registry service_registry; + static my_interface_mapping interface_mapping; + + public: + /** + De-initializes registry, other structures. + */ + static void deinit(); + + /** + Gets current reference count for a Service Implementation related to the + specified pointer to the interface structure. Assumes caller has at least + a read lock on the Registry. + + @param interface A pointer to the interface structure of the Service + Implementation to get reference count of. + @return A current reference count for specified Service Implementation. + Returns 0 in case there is no such interface or it is not referenced. + */ + static uint64_t get_service_implementation_reference_count( + my_h_service interface); + + /** + Finds and acquires a Service by name. A name of the Service or the Service + Implementation can be specified. In case of the Service name, the default + Service Implementation for Service specified will be returned. Assumes + caller has at least a read lock on the Registry. + + @param service_name Name of Service or Service Implementation to acquire. + @param [out] out_service Pointer to Service handle to set acquired Service. + @return Status of performed operation + @retval false success + @retval true failure + */ + static bool acquire_nolock(const char *service_name, + my_h_service *out_service); + + /** + Releases the Service Implementation previously acquired. After the call to + this method the usage of the Service Implementation handle will lead to + unpredicted results. Assumes caller has at least a read lock on the + Registry. + + @param service Service Implementation handle of already acquired Service. + @return Status of performed operation + @retval false success + @retval true failure + */ + static bool release_nolock(my_h_service service); + + /** + Registers a new Service Implementation. If it is the first Service + Implementation for the specified Service then it is made a default one. + Assumes caller has a write lock on the Registry. + + @param service_implementation_name Name of the Service Implementation to + register. + @param ptr Pointer to the Service Implementation structure. + @return Status of performed operation + @retval false success + @retval true failure + */ + static bool register_service_nolock(const char *service_implementation_name, + my_h_service ptr); + + /** + Removes previously registered Service Implementation from registry. If it is + the default one for specified Service then any one still registered is made + default. If there is no other, the default entry is removed from the + Registry too. Assumes caller has a write lock on the Registry. + + @param service_implementation_name Name of the Service Implementation to + unregister. + @return Status of performed operation + @retval false success + @retval true Failure. May happen when Service is still being referenced. + */ + static bool unregister_nolock(const char *service_implementation_name); + + /* Service Implementations */ + /** + Finds and acquires a Service by name. A name of the Service or the Service + Implementation can be specified. In case of the Service name, the default + Service Implementation for Service specified will be returned. + + This does not take any lock on the registry. It must not be used unless + absolutely necessary. Use the mysql_registry_imp version instead. + + @param service_name Name of Service or Service Implementation to acquire. + @param [out] out_service Pointer to Service handle to set acquired Service. + @return Status of performed operation + @retval false success + @retval true failure + */ + static DEFINE_BOOL_METHOD(acquire, (const char *service_name, + my_h_service *out_service)); + + /** + Finds a Service by name. If there is a Service Implementation with the same + Component part of name as the input Service then the found Service is + returned. + + This does not take any lock on the registry. It must not be used unless + absolutely necessary. Use the mysql_registry_imp version instead. + + @param service_name Name of Service or Service Implementation to acquire. + @param service Service handle already acquired Service Implementation. + @param [out] out_service Pointer to Service Implementation handle to set + acquired Service Implementation. + @return Status of performed operation + @retval false success + @retval true failure + */ + static DEFINE_BOOL_METHOD(acquire_related, + (const char *service_name, my_h_service service, + my_h_service *out_service)); + + /** + Releases the Service Implementation previously acquired. After the call to + this method the usage of the Service Implementation handle will lead to + unpredicted results. + + This does not take any lock on the registry. It must not be used unless + absolutely necessary. Use the mysql_registry_imp version instead. + + @param service Service Implementation handle of already acquired Service. + @return Status of performed operation + @retval false success + @retval true failure + */ + static DEFINE_BOOL_METHOD(release, (my_h_service service)); + + /** + Registers a new Service Implementation. If it is the first Service + Implementation for the specified Service then it is made a default one. + + This does not take any lock on the registry. It must not be used unless + absolutely necessary. Use the mysql_registry_imp version instead. + + @param service_implementation_name Name of the Service Implementation to + register. + @param ptr Pointer to the Service Implementation structure. + @return Status of performed operation + @retval false success + @retval true failure + */ + static DEFINE_BOOL_METHOD(register_service, + (const char *service_implementation_name, + my_h_service ptr)); + + /** + Removes previously registered Service Implementation from registry. If it is + the default one for specified Service then any one still registered is made + default. If there is no other, the default entry is removed from the + Registry too. + + This does not take any lock on the registry. It must not be used unless + absolutely necessary. Use the mysql_registry_imp version instead. + + @param service_implementation_name Name of the Service Implementation to + unregister. + @return Status of performed operation + @retval false success + @retval true Failure. May happen when Service is still being referenced. + */ + static DEFINE_BOOL_METHOD(unregister, + (const char *service_implementation_name)); + + /** + Sets new default Service Implementation for corresponding Service name. + + This does not take any lock on the registry. It must not be used unless + absolutely necessary. Use the mysql_registry_imp version instead. + + @param service_implementation_name Name of the Service Implementation to + set as default one. + @return Status of performed operation + @retval false success + @retval true failure + */ + static DEFINE_BOOL_METHOD(set_default, + (const char *service_implementation_name)); + + private: + /** + Finds a Service Implementation data structure based on the pointer to + interface struct supplied. Assumes caller has at least a read lock on the + Registry. + + @param interface A pointer to the interface structure of the Service + Implementation to look for. + @return A pointer to respective Service Implementation data structure, or + NULL if no such interface pointer is registered within the Registry. + */ + static mysql_service_implementation *get_service_implementation_by_interface( + my_h_service interface); +}; +#endif /* MYSQL_SERVER_REGISTRY_NO_LOCK_H */ diff --git a/include/mysql/components/services/mysql_command_services.h b/include/mysql/components/services/mysql_command_services.h index d22c37cbd09b..acf63225f651 100644 --- a/include/mysql/components/services/mysql_command_services.h +++ b/include/mysql/components/services/mysql_command_services.h @@ -61,7 +61,8 @@ enum mysql_command_option { MYSQL_COMMAND_PROTOCOL, MYSQL_COMMAND_USER_NAME, MYSQL_COMMAND_HOST_NAME, - MYSQL_COMMAND_TCPIP_PORT + MYSQL_COMMAND_TCPIP_PORT, + MYSQL_NO_LOCK_REGISTRY }; /** diff --git a/sql/mysqld.cc b/sql/mysqld.cc index bca65b96867e..bec2c555c5fb 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -2054,6 +2054,7 @@ SERVICE_TYPE(mysql_runtime_error) * error_service; SERVICE_TYPE(mysql_psi_system_v1) * system_service; SERVICE_TYPE(mysql_rwlock_v1) * rwlock_service; SERVICE_TYPE_NO_CONST(registry) * srv_registry; +SERVICE_TYPE_NO_CONST(registry) * srv_registry_no_lock; SERVICE_TYPE(dynamic_loader_scheme_file) * scheme_file_srv; using loader_type_t = SERVICE_TYPE_NO_CONST(dynamic_loader); using runtime_error_type_t = SERVICE_TYPE_NO_CONST(mysql_runtime_error); @@ -2087,6 +2088,10 @@ static bool component_infrastructure_init() { LogErr(ERROR_LEVEL, ER_COMPONENTS_INFRASTRUCTURE_BOOTSTRAP); return true; } + srv_registry->acquire( + "registry.mysql_minimal_chassis_no_lock", + reinterpret_cast(&srv_registry_no_lock)); + /* Here minimal_chassis dynamic_loader_scheme_file service has to be acquired */ srv_registry->acquire( @@ -2192,6 +2197,7 @@ static bool component_infrastructure_deinit() { srv_registry->release(reinterpret_cast( const_cast(scheme_file_srv))); + srv_registry->release(reinterpret_cast(srv_registry_no_lock)); srv_registry->release(reinterpret_cast( const_cast(dynamic_loader_srv))); srv_registry->release(reinterpret_cast( diff --git a/sql/mysqld.h b/sql/mysqld.h index f4dd2c2817d4..aa5464c78e78 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -905,6 +905,7 @@ extern mysql_component_t mysql_component_performance_schema; /* This variable is a registry handler, defined in mysql_server component and used as a output parameter for minimal chassis. */ extern SERVICE_TYPE_NO_CONST(registry) * srv_registry; +extern SERVICE_TYPE_NO_CONST(registry) * srv_registry_no_lock; /* These global variables which are defined and used in mysql_server component */ extern SERVICE_TYPE(dynamic_loader_scheme_file) * scheme_file_srv; diff --git a/sql/server_component/mysql_command_backend.cc b/sql/server_component/mysql_command_backend.cc index b962b1c34210..ecf2c66e4286 100644 --- a/sql/server_component/mysql_command_backend.cc +++ b/sql/server_component/mysql_command_backend.cc @@ -37,6 +37,7 @@ #include "sql/srv_session.h" extern SERVICE_TYPE_NO_CONST(registry) * srv_registry; +extern SERVICE_TYPE_NO_CONST(registry) * srv_registry_no_lock; namespace cs { @@ -63,60 +64,11 @@ MYSQL_METHODS mysql_methods = { nullptr, /* read_change_user_result_nonblocking */ }; -mysql_state_machine_status cssm_begin_connect(mysql_async_connect *ctx) { - MYSQL *mysql = ctx->mysql; - auto mcs_extn = MYSQL_COMMAND_SERVICE_EXTN(mysql); - assert(mcs_extn); - const char *host = ctx->host; - const char *user = ctx->user; - const char *db = ctx->db; - MYSQL_THD thd; +static mysql_state_machine_status acquire_services( + mysql_command_consumer_refs *consumer_refs, + mysql_service_registry_t *srv_registry) { my_h_service h_command_consumer = nullptr; my_h_service h_command_consumer_srv = nullptr; - - MYSQL_SESSION mysql_session = nullptr; - if (mcs_extn->mcs_thd == nullptr || mcs_extn->session_svc == nullptr) { - /* - Avoid possibility of nested txn in the current thd. - If it is called, for example from a UDF. - */ - my_service service( - "mysql_admin_session.mysql_server", srv_registry); - if (service.is_valid()) { - mysql_session = service->open(nullptr, ctx); - if (mysql_session == nullptr) return STATE_MACHINE_FAILED; - } else - return STATE_MACHINE_FAILED; - thd = mysql_session->get_thd(); - mcs_extn->is_thd_associated = false; - Security_context_handle sc; - if (mysql_security_context_imp::get(thd, &sc)) return STATE_MACHINE_FAILED; - if (mysql_security_context_imp::lookup(sc, user, host, nullptr, db)) - return STATE_MACHINE_FAILED; - mcs_extn->mcs_thd = thd; - mysql->thd = thd; - mcs_extn->session_svc = mysql_session; - } else { - mysql->thd = reinterpret_cast(mcs_extn->mcs_thd); - } - /* - These references might be created in mysql_command_services_imp::set api. - If not, we will create here. - */ - if (mcs_extn->command_consumer_services == nullptr) { - /* - Provide default implementations for mysql command consumer services - and will be released in close() api. - */ - mcs_extn->command_consumer_services = new mysql_command_consumer_refs(); - } - mysql_command_consumer_refs *consumer_refs = - (mysql_command_consumer_refs *)mcs_extn->command_consumer_services; - /* The above new allocation failed */ - if (consumer_refs == nullptr) return STATE_MACHINE_FAILED; - /* If the service is not acquired by mysql_command_services_imp::set api, - then it will be acquired below. The same will be applicable for all - other below services. */ if (consumer_refs->factory_srv == nullptr) { if (srv_registry->acquire("mysql_text_consumer_factory_v1.mysql_server", &h_command_consumer)) @@ -237,6 +189,68 @@ mysql_state_machine_status cssm_begin_connect(mysql_async_connect *ctx) { mysql_text_consumer_client_capabilities_v1) *>( h_command_consumer_srv); } + return STATE_MACHINE_DONE; +} + +mysql_state_machine_status cssm_begin_connect(mysql_async_connect *ctx) { + MYSQL *mysql = ctx->mysql; + Mysql_handle mysql_handle; + mysql_handle.mysql = mysql; + auto mcs_extn = MYSQL_COMMAND_SERVICE_EXTN(mysql); + assert(mcs_extn); + const char *host = ctx->host; + const char *user = ctx->user; + const char *db = ctx->db; + MYSQL_THD thd; + bool no_lock_registry = false; + MYSQL_SESSION mysql_session = nullptr; + + if (mysql_command_services_imp::get( + (MYSQL_H)&mysql_handle, MYSQL_NO_LOCK_REGISTRY, &no_lock_registry)) + return STATE_MACHINE_FAILED; + mysql_service_registry_t *registry_service = + no_lock_registry ? srv_registry_no_lock : srv_registry; + + if (mcs_extn->mcs_thd == nullptr || mcs_extn->session_svc == nullptr) { + /* + Avoid possibility of nested txn in the current thd. + If it is called, for example from a UDF. + */ + my_service service( + "mysql_admin_session.mysql_server", registry_service); + if (service.is_valid()) mysql_session = service->open(nullptr, ctx); + if (mysql_session == nullptr) return STATE_MACHINE_FAILED; + thd = mysql_session->get_thd(); + mcs_extn->is_thd_associated = false; + Security_context_handle sc; + if (mysql_security_context_imp::get(thd, &sc)) return STATE_MACHINE_FAILED; + if (mysql_security_context_imp::lookup(sc, user, host, nullptr, db)) + return STATE_MACHINE_FAILED; + mcs_extn->mcs_thd = thd; + mysql->thd = thd; + mcs_extn->session_svc = mysql_session; + } else { + mysql->thd = reinterpret_cast(mcs_extn->mcs_thd); + } + /* + These references might be created in mysql_command_services_imp::set api. + If not, we will create here. + */ + if (mcs_extn->command_consumer_services == nullptr) { + /* + Provide default implementations for mysql command consumer services + and will be released in close() api. + */ + mcs_extn->command_consumer_services = new mysql_command_consumer_refs(); + } + mysql_command_consumer_refs *consumer_refs = + (mysql_command_consumer_refs *)mcs_extn->command_consumer_services; + /* The above new allocation failed */ + if (consumer_refs == nullptr) return STATE_MACHINE_FAILED; + /* If the services are not acquired by mysql_command_services_imp::set api, + then it will be acquired. */ + auto status = acquire_services(consumer_refs, registry_service); + if (status == STATE_MACHINE_FAILED) return status; mysql->client_flag = 0; /* For handshake */ mysql->server_status = SERVER_STATUS_AUTOCOMMIT; return STATE_MACHINE_DONE; diff --git a/sql/server_component/mysql_command_services_imp.cc b/sql/server_component/mysql_command_services_imp.cc index 23c87b9ed09b..5490dc5f513b 100644 --- a/sql/server_component/mysql_command_services_imp.cc +++ b/sql/server_component/mysql_command_services_imp.cc @@ -160,6 +160,81 @@ DEFINE_BOOL_METHOD(mysql_command_services_imp::reset, (MYSQL_H mysql_h)) { } } +/** + Release services. + + @param[in] consumer_refs A valid mysql_command_consumer_refs object. + @param[in] mcs_ext A valid mysql_command_service_extn object. + @param[in] srv_registry Registry service pointer. +*/ +static void release_services(mysql_command_consumer_refs *consumer_refs, + mysql_command_service_extn *mcs_ext, + mysql_service_registry_t *srv_registry) { + if (consumer_refs) { + if (consumer_refs->factory_srv) { + /* This service call is used to free the memory, the allocation + was happened through factory_srv->start() service api. */ + consumer_refs->factory_srv->end( + reinterpret_cast(mcs_ext->consumer_srv_data)); + srv_registry->release(reinterpret_cast( + const_cast( + consumer_refs->factory_srv))); + } + if (consumer_refs->metadata_srv) + srv_registry->release(reinterpret_cast( + const_cast( + consumer_refs->metadata_srv))); + if (consumer_refs->row_factory_srv) + srv_registry->release(reinterpret_cast( + const_cast( + consumer_refs->row_factory_srv))); + if (consumer_refs->error_srv) + srv_registry->release(reinterpret_cast( + const_cast( + consumer_refs->error_srv))); + if (consumer_refs->get_null_srv) + srv_registry->release(reinterpret_cast( + const_cast( + consumer_refs->get_null_srv))); + if (consumer_refs->get_integer_srv) + srv_registry->release(reinterpret_cast( + const_cast( + consumer_refs->get_integer_srv))); + if (consumer_refs->get_longlong_srv) + srv_registry->release(reinterpret_cast( + const_cast( + consumer_refs->get_longlong_srv))); + if (consumer_refs->get_decimal_srv) + srv_registry->release(reinterpret_cast( + const_cast( + consumer_refs->get_decimal_srv))); + if (consumer_refs->get_double_srv) + srv_registry->release(reinterpret_cast( + const_cast( + consumer_refs->get_double_srv))); + if (consumer_refs->get_date_time_srv) + srv_registry->release(reinterpret_cast( + const_cast( + consumer_refs->get_date_time_srv))); + if (consumer_refs->get_string_srv) + srv_registry->release(reinterpret_cast( + const_cast( + consumer_refs->get_string_srv))); + if (consumer_refs->client_capabilities_srv) + srv_registry->release(reinterpret_cast( + const_cast( + consumer_refs->client_capabilities_srv))); + } +} + /** Calls mysql_close api to closes a server connection. @@ -176,76 +251,14 @@ DEFINE_BOOL_METHOD(mysql_command_services_imp::close, (MYSQL_H mysql_h)) { auto mcs_ext = MYSQL_COMMAND_SERVICE_EXTN(mysql); mysql_command_consumer_refs *consumer_refs = (mysql_command_consumer_refs *)(mcs_ext->command_consumer_services); + bool no_lock_registry = false; + if (get(mysql_h, MYSQL_NO_LOCK_REGISTRY, &no_lock_registry)) return true; + no_lock_registry + ? release_services(consumer_refs, mcs_ext, srv_registry_no_lock) + : release_services(consumer_refs, mcs_ext, srv_registry); + delete consumer_refs; + consumer_refs = nullptr; - if (consumer_refs) { - if (consumer_refs->factory_srv) { - /* This service call is used to free the memory, the allocation - was happened through factory_srv->start() service api. */ - consumer_refs->factory_srv->end( - reinterpret_cast(mcs_ext->consumer_srv_data)); - srv_registry->release(reinterpret_cast( - const_cast( - consumer_refs->factory_srv))); - } - if (consumer_refs->metadata_srv) - srv_registry->release(reinterpret_cast( - const_cast( - consumer_refs->metadata_srv))); - if (consumer_refs->row_factory_srv) - srv_registry->release(reinterpret_cast( - const_cast( - consumer_refs->row_factory_srv))); - if (consumer_refs->error_srv) - srv_registry->release(reinterpret_cast( - const_cast( - consumer_refs->error_srv))); - if (consumer_refs->get_null_srv) - srv_registry->release(reinterpret_cast( - const_cast( - consumer_refs->get_null_srv))); - if (consumer_refs->get_integer_srv) - srv_registry->release(reinterpret_cast( - const_cast( - consumer_refs->get_integer_srv))); - if (consumer_refs->get_longlong_srv) - srv_registry->release(reinterpret_cast( - const_cast( - consumer_refs->get_longlong_srv))); - if (consumer_refs->get_decimal_srv) - srv_registry->release(reinterpret_cast( - const_cast( - consumer_refs->get_decimal_srv))); - if (consumer_refs->get_double_srv) - srv_registry->release(reinterpret_cast( - const_cast( - consumer_refs->get_double_srv))); - if (consumer_refs->get_date_time_srv) - srv_registry->release(reinterpret_cast( - const_cast( - consumer_refs->get_date_time_srv))); - if (consumer_refs->get_string_srv) - srv_registry->release(reinterpret_cast( - const_cast( - consumer_refs->get_string_srv))); - if (consumer_refs->client_capabilities_srv) - srv_registry->release(reinterpret_cast( - const_cast( - consumer_refs->client_capabilities_srv))); - - delete consumer_refs; - consumer_refs = nullptr; - } if (mcs_ext->is_thd_associated) delete (mcs_ext->session_svc); else { @@ -366,6 +379,10 @@ const char * |MYSQL_COMMAND_HOST_NAME |The host name to use to | | |security context. | --------------+-------------------------------+--------------------------------+ int |MYSQL_COMMAND_TCPIP_PORT |The port to use to connect. | +--------------+-------------------------------+--------------------------------+ +bool |MYSQL_NO_LOCK_REGISTRY |Flag to use the _no_lock | + | |implementation of registry | + | |service. | --------------+-------------------------------+--------------------------------+ @note For the other mysql client options it calls the mysql_options api from @@ -693,6 +710,9 @@ DEFINE_BOOL_METHOD(mysql_command_services_imp::set, mcs_ext->mcs_tcpip_port = *static_cast(arg); } break; + case MYSQL_NO_LOCK_REGISTRY: + mcs_ext->no_lock_registry = static_cast(arg); + break; default: if (mysql_options(m_handle->mysql, static_cast(option), arg) != 0) @@ -721,10 +741,18 @@ DEFINE_BOOL_METHOD(mysql_command_services_imp::get, (MYSQL_H mysql_h, int option, const void *arg)) { try { Mysql_handle *m_handle = reinterpret_cast(mysql_h); - if (m_handle == nullptr) return true; - if (mysql_get_option(m_handle->mysql, - static_cast(option), arg) != 0) - return true; + auto mcs_ext = MYSQL_COMMAND_SERVICE_EXTN(m_handle->mysql); + if (m_handle == nullptr || !arg) return true; + switch (option) { + case MYSQL_NO_LOCK_REGISTRY: + *(const_cast(static_cast(arg))) = + mcs_ext->no_lock_registry; + break; + default: + if (mysql_get_option(m_handle->mysql, + static_cast(option), arg) != 0) + return true; + } } catch (...) { mysql_components_handle_std_exception(__func__); return true; diff --git a/sql/server_component/mysql_command_services_imp.h b/sql/server_component/mysql_command_services_imp.h index 658faa3337f9..aaca94bb4e77 100644 --- a/sql/server_component/mysql_command_services_imp.h +++ b/sql/server_component/mysql_command_services_imp.h @@ -72,6 +72,7 @@ struct mysql_command_service_extn { int mcs_tcpip_port; const char *mcs_db = nullptr; uint32_t mcs_client_flag = 0; + bool no_lock_registry = false; }; #define MYSQL_COMMAND_SERVICE_EXTN(H) \ From b7cca05eb2bd7a022075acb289e88fba8eaf9e56 Mon Sep 17 00:00:00 2001 From: Yura Sorokin Date: Mon, 18 Nov 2024 18:22:37 +0100 Subject: [PATCH 03/11] Re-applied temporarily partially reverted fix for PS-9453 Re-applied the fix for PS-9453 "percona_telemetry causes a long wait on COND_thd_list due to the absence of the root user" (commit e53363d) partially reverted previously. After changes in 'cssm_begin_connect()' instead of cherry-picking from 8.0 branch the modified patch from 8.4 was taken. This is a finalization step of for cherry-picking Bug #34741098 "component::deinit() will block if calling any registry update APIs" (commit mysql/mysql-server@d39330f). --- sql/server_component/mysql_command_backend.cc | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/sql/server_component/mysql_command_backend.cc b/sql/server_component/mysql_command_backend.cc index ecf2c66e4286..b29cf1507a6c 100644 --- a/sql/server_component/mysql_command_backend.cc +++ b/sql/server_component/mysql_command_backend.cc @@ -211,6 +211,20 @@ mysql_state_machine_status cssm_begin_connect(mysql_async_connect *ctx) { mysql_service_registry_t *registry_service = no_lock_registry ? srv_registry_no_lock : srv_registry; + /* We need to handle the failure in this function. + Setting mcs_extn->session_svc right after session open is not enough + to handle user lookup errors (and following errors as well) + because in case of this function returns error, mysql->extension is + cleaned up immediately by the caller. The caller does not take care of + session_svc, because it is not aware of this structure. + */ + std::shared_ptr mysql_session_close_guard( + &mysql_session, [mcs_extn](MYSQL_SESSION *mysql_session_ptr) { + if (*mysql_session_ptr == nullptr) return; + mcs_extn->session_svc = nullptr; + srv_session_close(*mysql_session_ptr); + }); + if (mcs_extn->mcs_thd == nullptr || mcs_extn->session_svc == nullptr) { /* Avoid possibility of nested txn in the current thd. @@ -253,6 +267,7 @@ mysql_state_machine_status cssm_begin_connect(mysql_async_connect *ctx) { if (status == STATE_MACHINE_FAILED) return status; mysql->client_flag = 0; /* For handshake */ mysql->server_status = SERVER_STATUS_AUTOCOMMIT; + mysql_session = nullptr; // disable delete quard return STATE_MACHINE_DONE; } From 1e12635b88f778c9710ca3dec97492ab7b3bcd93 Mon Sep 17 00:00:00 2001 From: Yura Sorokin Date: Fri, 22 Nov 2024 00:07:40 +0100 Subject: [PATCH 04/11] PS-9551 fix: Setting MYSQL_COMMAND_LOCAL_THD_HANDLE may crash the server https://perconadev.atlassian.net/browse/PS-9551 Fixed problem in 'mysql_command_services_imp::set()'. When user sets the 'MYSQL_COMMAND_LOCAL_THD_HANDLE' option very early after server startup (when 'srv_session_server_is_available()' still returns false), 'service->open(nullptr, nullptr)' may return nullptr and it is unsafe to use it afterwards. Fixed by checking for nullness and returning earlier. --- sql/server_component/mysql_command_services_imp.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sql/server_component/mysql_command_services_imp.cc b/sql/server_component/mysql_command_services_imp.cc index 5490dc5f513b..46f459576a82 100644 --- a/sql/server_component/mysql_command_services_imp.cc +++ b/sql/server_component/mysql_command_services_imp.cc @@ -660,6 +660,8 @@ DEFINE_BOOL_METHOD(mysql_command_services_imp::set, mysql_session = service->open(nullptr, nullptr); else return true; + if (mysql_session == nullptr) + return true; thd = mysql_session->get_thd(); mcs_ext->is_thd_associated = false; Security_context_handle sc; From 281e4bf6f291a42360dcd53e330e44982691d3f9 Mon Sep 17 00:00:00 2001 From: Yura Sorokin Date: Fri, 15 Nov 2024 00:37:33 +0100 Subject: [PATCH 05/11] PS-9537 fix: Existing connection cannot be reused to run multiple queries in Command Services https://perconadev.atlassian.net/browse/PS-9537 Fixed problem with re-using the same connection created via 'mysql_command_factory->init()' / 'mysql_command_factory->connect()' in multiple calls to' mysql_command_query->query()'. The problem seems to be a regression introduced by upstream in their fix for Bug #34323788 "ASAN memory leaks reported by test_mysql_command_services_component.test" (commit mysql/mysql-server@5dc1a14). Inside 'csi_advanced_command()' 'mcs_extn->consumer_srv_data' when is not nullptr cannot be simply re-used as its `mcs_extn->data` member has already been set to nullptr by `std::exchange()` inside `csi_read_rows()`. Fixed by calling 'factory_srv->end()' and 'factory_srv->start()' in this case. --- sql/server_component/mysql_command_backend.cc | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/sql/server_component/mysql_command_backend.cc b/sql/server_component/mysql_command_backend.cc index b29cf1507a6c..e72bdf778764 100644 --- a/sql/server_component/mysql_command_backend.cc +++ b/sql/server_component/mysql_command_backend.cc @@ -306,10 +306,13 @@ bool csi_advanced_command(MYSQL *mysql, enum enum_server_command command, } mysql_handle.mysql = mysql; - if (mcs_extn->consumer_srv_data != nullptr) - srv_ctx_h = reinterpret_cast(mcs_extn->consumer_srv_data); - else if (((class mysql_command_consumer_refs *)(command_consumer_srv)) - ->factory_srv->start(&srv_ctx_h, (MYSQL_H *)&mysql_handle)) { + if (mcs_extn->consumer_srv_data != nullptr) { + ((class mysql_command_consumer_refs *)(command_consumer_srv)) + ->factory_srv->end( + reinterpret_cast(mcs_extn->consumer_srv_data)); + } + if (((class mysql_command_consumer_refs *)(command_consumer_srv)) + ->factory_srv->start(&srv_ctx_h, (MYSQL_H *)&mysql_handle)) { sprintf(*err_msg, "Could not create %s service", "mysql_text_consumer_factory_v1"); goto error; From cdcd38ae81d0178f84094022b2a5fd782fc8adce Mon Sep 17 00:00:00 2001 From: Yura Sorokin Date: Sat, 11 Jan 2025 02:39:13 +0100 Subject: [PATCH 06/11] UDF Wrappers headers made clang-tidy warning-free Removed some outdated Clang 5 warning suppressions. --- include/mysqlpp/udf_context.hpp | 2 +- .../mysqlpp/udf_context_charset_extension.hpp | 20 ++-- include/mysqlpp/udf_registration.hpp | 6 +- include/mysqlpp/udf_traits.hpp | 3 +- include/mysqlpp/udf_wrappers.hpp | 103 ++++++++---------- 5 files changed, 68 insertions(+), 66 deletions(-) diff --git a/include/mysqlpp/udf_context.hpp b/include/mysqlpp/udf_context.hpp index d4134cad2dd4..295ec0c255ec 100644 --- a/include/mysqlpp/udf_context.hpp +++ b/include/mysqlpp/udf_context.hpp @@ -94,7 +94,7 @@ class udf_context { } void mark_arg_nullable(std::size_t index, bool nullable) noexcept { - args_->maybe_null[index] = (nullable ? 1 : 0); + args_->maybe_null[index] = (nullable ? '\1' : '\0'); } void mark_result_nullable(bool nullable) noexcept { diff --git a/include/mysqlpp/udf_context_charset_extension.hpp b/include/mysqlpp/udf_context_charset_extension.hpp index b6498b69b163..acc037c89ad2 100644 --- a/include/mysqlpp/udf_context_charset_extension.hpp +++ b/include/mysqlpp/udf_context_charset_extension.hpp @@ -46,16 +46,17 @@ class udf_context_charset_extension { const char *get_return_value_charset(const udf_context &ctx) const { void *output = nullptr; if (udf_metadata_service_->result_get(ctx.initid_, charset_attribute_name, - &output)) + &output) != 0) throw std::runtime_error{"cannot get return value character set"}; return static_cast(output); } void set_return_value_charset(udf_context &ctx, const char *charset) const { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) void *cs = const_cast(charset); if (udf_metadata_service_->result_set(ctx.initid_, charset_attribute_name, - cs)) + cs) != 0) throw std::runtime_error{"cannot set return value character set"}; } @@ -66,7 +67,7 @@ class udf_context_charset_extension { "cannot get character set of a non-string argument"}; if (udf_metadata_service_->argument_get(ctx.args_, charset_attribute_name, - index, &output)) + index, &output) != 0) throw std::runtime_error{"cannot get argument character set"}; return static_cast(output); @@ -74,20 +75,21 @@ class udf_context_charset_extension { void set_arg_value_charset(udf_context &ctx, std::size_t index, const char *charset) const { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) void *cs = const_cast(charset); if (ctx.args_->arg_type[index] != STRING_RESULT) throw std::runtime_error{ "cannot set character set of a non-string argument"}; if (udf_metadata_service_->argument_set(ctx.args_, charset_attribute_name, - index, cs)) + index, cs) != 0) throw std::runtime_error{"cannot set argument value character set"}; } const char *get_return_value_collation(const udf_context &ctx) const { void *output = nullptr; if (udf_metadata_service_->result_get(ctx.initid_, collation_attribute_name, - &output)) + &output) != 0) throw std::runtime_error{"cannot get return value collation"}; return static_cast(output); @@ -95,9 +97,10 @@ class udf_context_charset_extension { void set_return_value_collation(udf_context &ctx, const char *collation) const { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) void *cs = const_cast(collation); if (udf_metadata_service_->result_set(ctx.initid_, collation_attribute_name, - cs)) + cs) != 0) throw std::runtime_error{"cannot set return value collation"}; } @@ -108,7 +111,7 @@ class udf_context_charset_extension { throw std::runtime_error{"cannot get collation of a non-string argument"}; if (udf_metadata_service_->argument_get(ctx.args_, collation_attribute_name, - index, &output)) + index, &output) != 0) throw std::runtime_error{"cannot get argument collation"}; return static_cast(output); @@ -116,12 +119,13 @@ class udf_context_charset_extension { void set_arg_value_collation(udf_context &ctx, std::size_t index, const char *collation) const { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) void *cs = const_cast(collation); if (ctx.args_->arg_type[index] != STRING_RESULT) throw std::runtime_error{"cannot set collation of a non-string argument"}; if (udf_metadata_service_->argument_set(ctx.args_, collation_attribute_name, - index, cs)) + index, cs) != 0) throw std::runtime_error{"cannot set argument value collation"}; } diff --git a/include/mysqlpp/udf_registration.hpp b/include/mysqlpp/udf_registration.hpp index aef40e8c38f9..82e4cf903ae0 100644 --- a/include/mysqlpp/udf_registration.hpp +++ b/include/mysqlpp/udf_registration.hpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -127,9 +128,11 @@ void unregister_udfs(SERVICE_TYPE(udf_registration) * service, } // namespace mysqlpp +// NOLINTBEGIN(cppcoreguidelines-macro-usage) #define DECLARE_UDF_INFO(NAME, TYPE) \ mysqlpp::udf_info { \ - #NAME, TYPE, (Udf_func_any)&NAME, &NAME##_init, &NAME##_deinit \ + #NAME, TYPE, (Udf_func_any)std::addressof(NAME), \ + std::addressof(NAME##_init), std::addressof(NAME##_deinit) \ } // A simplified version of the DECLARE_UDF_INFO macro that relies on the @@ -138,5 +141,6 @@ void unregister_udfs(SERVICE_TYPE(udf_registration) * service, #define DECLARE_UDF_INFO_AUTO(NAME) \ DECLARE_UDF_INFO(NAME, \ ::mysqlpp::udf_impl_meta_info::item_result) +// NOLINTEND(cppcoreguidelines-macro-usage) #endif diff --git a/include/mysqlpp/udf_traits.hpp b/include/mysqlpp/udf_traits.hpp index 905c9dd4384a..c2b081bd0887 100644 --- a/include/mysqlpp/udf_traits.hpp +++ b/include/mysqlpp/udf_traits.hpp @@ -34,7 +34,8 @@ struct wrapped_t { template struct impl_with_mixin : public MixinType { - impl_with_mixin(udf_context &ctx) : MixinType{}, impl{ctx} {} + explicit impl_with_mixin(udf_context &ctx) : MixinType{}, impl{ctx} {} + // NOLINTNEXTLINE(misc-non-private-member-variables-in-classes) ImplType impl; }; diff --git a/include/mysqlpp/udf_wrappers.hpp b/include/mysqlpp/udf_wrappers.hpp index b53c3ced9a83..840d09941e1f 100644 --- a/include/mysqlpp/udf_wrappers.hpp +++ b/include/mysqlpp/udf_wrappers.hpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -54,7 +55,7 @@ class udf_base { static const char *get_function_label(std::string &buffer, const char *meta_name, item_result_type item_result) noexcept { - const char *res = ""; + const char *res{nullptr}; try { buffer = meta_name; buffer += '<'; @@ -63,6 +64,7 @@ class udf_base { buffer += '>'; res = buffer.c_str(); } catch (...) { + res = ""; } return res; } @@ -72,20 +74,11 @@ class udf_base { assert(error_reporter != nullptr); std::string buffer; try { - // The following suppression is needed exclusively for Clang 5.0 that - // has a bug in noexcept specification diagnostics. -#if defined(__clang__) && (__clang_major__ == 5) -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wexceptions" -#endif // Rethrowing the exception that was previously caught with // 'catch(...)' in one of the derived classes // This is done to write the sequence of the catch blocks // in one place. throw; -#if defined(__clang__) && (__clang_major__ == 5) -#pragma clang diagnostic pop -#endif } catch (const udf_exception &e) { if (e.has_error_code()) { auto error_code = e.get_error_code(); @@ -116,20 +109,11 @@ class udf_base { static void handle_init_exception(char *message, std::size_t message_size) noexcept { try { - // The following suppression is needed exclusively for Clang 5.0 that - // has a bug in noexcept specification diagnostics -#if defined(__clang__) && (__clang_major__ == 5) -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wexceptions" -#endif // Rethrowing the exception that was previously caught with // 'catch(...)' in one of the derived classes // This is done to write the sequence of the catch blocks // in one place. throw; -#if defined(__clang__) && (__clang_major__ == 5) -#pragma clang diagnostic pop -#endif } catch (const std::exception &e) { std::strncpy(message, e.what(), message_size); message[message_size - 1] = '\0'; @@ -155,18 +139,21 @@ class generic_udf_base : private udf_base { udf_context udf_ctx{initid, args}; extended_impl_t *impl = nullptr; try { + // NOLINTNEXTLINE(cppcoreguidelines-owning-memory) impl = new extended_impl_t{udf_ctx}; } catch (...) { handle_init_exception(message, MYSQL_ERRMSG_SIZE); return true; } + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) initid->ptr = reinterpret_cast(impl); return false; } static void deinit(UDF_INIT *initid) noexcept { + // NOLINTNEXTLINE(cppcoreguidelines-owning-memory) delete get_extended_impl_from_udf_initid(initid); } @@ -175,6 +162,7 @@ class generic_udf_base : private udf_base { static extended_impl_t *get_extended_impl_from_udf_initid( UDF_INIT *initid) noexcept { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) return reinterpret_cast(initid->ptr); } }; @@ -207,12 +195,11 @@ class generic_udf assert(udf_ctx.is_result_nullabale()); *is_null = 1; return nullptr; - } else { - *is_null = 0; - extended_impl.mixin = std::move(res.value()); - *length = extended_impl.mixin.size(); - return const_cast(extended_impl.mixin.c_str()); } + *is_null = 0; + extended_impl.mixin = std::move(*res); + *length = extended_impl.mixin.size(); + return const_cast(extended_impl.mixin.c_str()); } }; @@ -240,10 +227,9 @@ class generic_udf assert(udf_ctx.is_result_nullabale()); *is_null = 1; return 0.0; - } else { - *is_null = 0; - return res.value(); } + *is_null = 0; + return *res; } }; @@ -271,15 +257,15 @@ class generic_udf assert(udf_ctx.is_result_nullabale()); *is_null = 1; return 0; - } else { - *is_null = 0; - return res.value(); } + *is_null = 0; + return *res; } }; } // namespace mysqlpp +// NOLINTBEGIN(cppcoreguidelines-macro-usage) #define DECLARE_UDF_META_INFO(IMPL, RESULT_TYPE, NAME) \ namespace mysqlpp { \ template <> \ @@ -290,40 +276,44 @@ class generic_udf }; \ } -#define DECLARE_UDF_INIT(IMPL, RESULT_TYPE, NAME) \ - MYSQLPP_UDF_EXPORT \ - bool NAME##_init(UDF_INIT *initid, UDF_ARGS *args, char *message) { \ - static_assert(std::is_same_v, \ - "Invalid UDF init function signature"); \ - return mysqlpp::generic_udf::init(initid, args, \ - message); \ +#define DECLARE_UDF_INIT(IMPL, RESULT_TYPE, NAME) \ + MYSQLPP_UDF_EXPORT \ + bool NAME##_init(UDF_INIT *initid, UDF_ARGS *args, char *message) { \ + static_assert( \ + std::is_same_v, \ + "Invalid UDF init function signature"); \ + return mysqlpp::generic_udf::init(initid, args, \ + message); \ } -#define DECLARE_UDF_DEINIT(IMPL, RESULT_TYPE, NAME) \ - MYSQLPP_UDF_EXPORT \ - void NAME##_deinit(UDF_INIT *initid) { \ - static_assert(std::is_same_v, \ - "Invalid UDF deinit function signature"); \ - mysqlpp::generic_udf::deinit(initid); \ +#define DECLARE_UDF_DEINIT(IMPL, RESULT_TYPE, NAME) \ + MYSQLPP_UDF_EXPORT \ + void NAME##_deinit(UDF_INIT *initid) { \ + static_assert(std::is_same_v, \ + "Invalid UDF deinit function signature"); \ + mysqlpp::generic_udf::deinit(initid); \ } -#define DECLARE_UDF_STRING_FUNC(IMPL, NAME) \ - MYSQLPP_UDF_EXPORT \ - char *NAME(UDF_INIT *initid, UDF_ARGS *args, char *result, \ - unsigned long *length, unsigned char *is_null, \ - unsigned char *error) { \ - static_assert(std::is_same_v, \ - "Invalid string UDF function signature"); \ - return mysqlpp::generic_udf::func( \ - initid, args, result, length, is_null, error); \ +#define DECLARE_UDF_STRING_FUNC(IMPL, NAME) \ + MYSQLPP_UDF_EXPORT \ + char *NAME(UDF_INIT *initid, UDF_ARGS *args, char *result, \ + unsigned long *length, unsigned char *is_null, \ + unsigned char *error) { \ + static_assert( \ + std::is_same_v, \ + "Invalid string UDF function signature"); \ + return mysqlpp::generic_udf::func( \ + initid, args, result, length, is_null, error); \ } #define DECLARE_UDF_REAL_FUNC(IMPL, NAME) \ MYSQLPP_UDF_EXPORT \ double NAME(UDF_INIT *initid, UDF_ARGS *args, unsigned char *is_null, \ unsigned char *error) { \ - static_assert(std::is_same_v, \ - "Invalid real UDF function signature"); \ + static_assert( \ + std::is_same_v, \ + "Invalid real UDF function signature"); \ return mysqlpp::generic_udf::func(initid, args, \ is_null, error); \ } @@ -332,7 +322,8 @@ class generic_udf MYSQLPP_UDF_EXPORT \ long long NAME(UDF_INIT *initid, UDF_ARGS *args, unsigned char *is_null, \ unsigned char *error) { \ - static_assert(std::is_same::value, \ + static_assert(std::is_same::value, \ "Invalid int UDF function signature"); \ return mysqlpp::generic_udf::func(initid, args, is_null, \ error); \ @@ -363,4 +354,6 @@ class generic_udf #define DECLARE_REAL_UDF_AUTO(NAME) DECLARE_REAL_UDF(NAME##_impl, NAME) #define DECLARE_INT_UDF_AUTO(NAME) DECLARE_INT_UDF(NAME##_impl, NAME) +// NOLINTEND(cppcoreguidelines-macro-usage) + #endif From 4ef0bb3ea70dc8043a0c2e858fb4d1b5146af500 Mon Sep 17 00:00:00 2001 From: Yura Sorokin Date: Sat, 11 Jan 2025 02:44:13 +0100 Subject: [PATCH 07/11] PS-9148 feature: Add caching of dictionary table for component_masking_functions https://perconadev.atlassian.net/browse/PS-9148 - Added caching of mysql.masking_dictionaries table content. - Implemented masking_dictionaries_flush() UDF which flushes data from the masking dictionaries table to the memory cache. PS-9148 feature: Add masking_functions.masking_database sys var support https://perconadev.atlassian.net/browse/PS-9148 The masking_functions.masking_database system variable for the masking_functions component specifies database used for data masking dictionaries. PS-9148 feature: Implement dictionary flusher for masking_functions plugin https://perconadev.atlassian.net/browse/PS-9148 - Added component_masking.dictionaries_flush_interval_seconds system variable. - Added actual flusher thread. It periodically rereads content of dictionary table and updates in-memory cache. PS-9148 feature: Implemented hierarchical storage for dictionaries and terms https://perconadev.atlassian.net/browse/PS-9148 Introduced 'dictionary' and 'bookshelf' classes for storing terms on per-dictionary level. Reworked 'query_cache' to utilize these two new classes. PS-9148 feature: Minor refactoring to break dependencies https://perconadev.atlassian.net/browse/PS-9148 Introduced 'component_sys_variable_service_tuple' class for groupping comonent system variable registration services (supposed to be used with 'primitive_singleton' class template). 'query_cache' now expects 'query_builder' and 'flusher_interval_seconds' as its constructor's parameters. Eliminates custom MySQL types (like 'ulonglong') and its includes (like 'my_inttypes.h') from the publicly facing headers. 'query_cache' is now explicitly initialized / deinitialized in the component's 'init()'' / 'deinit()'' functions via 'primitive_singleton' interface. 'query_cache' helper thread-related methods made private. PS-9148 feature: Refactored usage of std::string_view for c-interfaces https://perconadev.atlassian.net/browse/PS-9148 As std::string_view::data() is not guaranteed to be null-terminated, it is not safe to use it in old c-functions accepting 'const char *'. Some constants converted to arrays of char 'const char buffer[]{"value"}'. PS-9148 feature: Implemented lazy query_cache initial population https://perconadev.atlassian.net/browse/PS-9148 'command_service_tuple' struct extended with one more member - 'field_info' service. Reworked 'query_cache' class: instead of loading terms from the database in constructor, this operation is now performed in first attempt to access one of the dictionary methods ('contains()' / 'get_random()' / 'remove()' / 'insert()'). This is done in order to overcome a limitation that does not allow 'mysql_command_query' service to be used from inside the componment initialization function. Fixed problem with 'm_dict_cache' shared pointer updated concurrently from different threads. Exceptions thrown from the cache loading function no longer escape the flusher thread. De-coupled 'sql_context' and 'bookshelf' classes: 'sql_context' now accepts a generic insertion callback that can be used to populate any type of containers. 'component_masking_functions.dictionary_operations' MTR test case extended with additional checks for flushed / unflushed dictionary cache. PS-9148 feature: Reworked dictionary / bookshelf thread-safety model https://perconadev.atlassian.net/browse/PS-9148 Both 'dictionary' and 'bookshelf' classes no longer include their own 'std::shared_mutex' to protect data. Instead, we now have a single 'std::shared_mutex' at the 'query_cache' level. The return value of the 'get_random()' method in both 'dictionary' and 'bookshelf' classes changed from 'optional_string' to 'std::string_view'. Empty (default constructed) 'std::string_view' is used as an indicator of an unsuccessful operation. 'get_random()' method in the 'query_cache' class still returns a string by value to avoid race conditions. Changed the behaviour of the 'sql_context::execute_dml()' method - it now throws when SQL errors (like "no table found", etc.) occur. PS-9148 feature: Fix masking functions flusher thread intialization https://perconadev.atlassian.net/browse/PS-9148 Added missing my_thread_attr_t initialization. PS-9148 feature: Decoupled threading and caching functionality in query_cache https://perconadev.atlassian.net/browse/PS-9148 Threading-related functionality extracted from the 'query_cache' class into a separate 'dictionary_flusher_thread' class. This new class now accepts an instance of existing 'query_cache' class as a parameter of its constructor in a form of shared pointer. Changed the way how these two objects are now initialized / deinitialized in 'component.cpp' ('component_init()' / 'component_deinit()' functions). PS-9148 feature: Refactored dictionary_flusher_thread https://perconadev.atlassian.net/browse/PS-9148 'dictionary_flusher_thread' class interface ('dictionary_flusher_thread.hpp') now includes no public / internal MySQL headers. Introduced internal 'thread_handler_context' class, which is supposed to be instantiated as the very first declaration of the MySQL thread handler function - it performs proper 'THD' object initialization in constructor and deinitialization in destructor. Introduced internal 'thread_attributes' class - an RAII wrapper over 'my_thread_attr_t' with proper initialization in constructor and deinitialization in destructor. Introduced internal 'jthread' class that similarly to 'std::jthread' from c++20 spawns a joinable thread in constructor and joins it in destructor. It expects only meaningful logic in a form of 'std::function' from the user and hides all the MySQL initialization / PSI registration boilerplate. It uses an instance of 'thread_attributes' class when spawns a thread. It also creates an instance of the 'thread_handler_context' class inside actual thread handler function ('jthread::raw_handler()'). This class also makes sure that no exception escapes actual thread handler function. Refactored error handling in 'component_init()'. Fixed problem with updating 'stopped_' variable (which the condition variable uses in its 'waitt_for()' method) without properly locking the mutex. Fixed instabilities in 'component_masking_functions.rpl_dictionaries_flush_interval' MTR test case. PS-9148 feature: Improved diagnostics (MySQL API error messages) in sql_context https://perconadev.atlassian.net/browse/PS-9148 'command_service_tuple' class extended with new 'error_info' member of type 'SERVICE_TYPE(mysql_command_error_info) *' that allows extracting MySQL error codes and messages. It is expected to be used inside 'sql_context' class methods. Reworked the way how exceptions are thrown from the 'sql_context' class methods - we now use 'raise_with_error_message()' helper method that throws an exception that incorporates MySQL client API error message extracted via added 'error_info' member of the `command_service_tuple` class. More meaningful error messages, which include underlying MySQL error descriptions, are now generated from inside the 'query_cache' class methods. Added new DBUG keywords and DEBUG_SYNC actions that allow control over Dictionary Flusher background thread actions. Added new 'component_masking_functions.flusher_thread_suspend_resume' MTR test case that checks for various race conditions between current session and Dictionary Flusher background thread. Improved 'component_masking_functions.rpl_dictionaries_flush_interval' MTR test case - pre-recorder values replaced with 'assert.inc'. PS-9148 feature: Refactored to avoid deadlock during UNINSTALL COMPONENT https://perconadev.atlassian.net/browse/PS-9148 'sql_context' class constructor now takes one extra parameter that allows to specify whether the user wants to set the 'MYSQL_NO_LOCK_REGISTRY' option or not. Introduced new abstract class 'basic_sql_context_builder' that is used to construct instances of the 'sql_context' class on demand. Also added two concrete implementations of this interface: 'default_sql_context_builder' and 'static_sql_context_builder'. The former just creates a new instance of the 'sql_context' class every time the 'build()' method is invoked. This implementation is used from inside the UDF function implementation methods. The latter creates an instance of the 'sql_context' class during the first call to the 'build()' method, saves it internally and returns this saved instance for each subsequent call to the 'build()' method. This implementation is used in the 'dictionary_flusher_thread'. Refactored 'query_cache' class: basic functionality that needs external 'basic_sql_context_builder' and 'query_builder' extracted into separate class 'sql_cache_core'. For convenience, added new 'sql_cache' class as a wrapper over existing 'sql_cache_code', 'basic_sql_context_builder' and 'query_builder'. The same 'sql_cache_core' can be shared between multiple instances of the `sql_cache`. This allowed to make sure that `dictionary_flusher_thread` uses a dedicated long-living instance of the 'sql_context' class (with 'MYSQL_NO_LOCK_REGISTRY' enabled) and does not cause deadlocks during 'UNINSTALL COMPONENT'. See Bug #34741098 "component::deinit() will block if calling any registry" (commit mysql/mysql-server@d39330f) for more details. As the 'sql_context' connection in the 'dictionary_flusher_thread' is now a long-living one, it is now shown in the output of the 'SHOW PROCESSLIST' statement. Modified 'count_sessions.inc' / 'wait_until_count_sessions.inc' logic inside 'component_masking_functions.flusher_thread_suspend_resume' MTR test case to reflect these changes. PS-9148 feature: Refactored dictionary flusher thread startup / termination https://perconadev.atlassian.net/browse/PS-9148 'command_service_tuple' extended with one more member 'thread' which is used to initialize / deinitialize threads intended to be used as MySQL threads. 'sql_context' class constructor now takes one extra parameter that allows to specify whether the user wants to associate a new session (including an instance of the 'THD' class) with the calling thread. Internally it is done by setting the 'MYSQL_COMMAND_LOCAL_THD_HANDLE' option to nullptr. Also 'sql_context' now tries to open connections on behalf of the internal predefined 'mysql.session' MySQL user (instead of 'root'). Reworked 'static_sql_context_builder' class - it now creates a shared "static" instance of he 'sql_context' class inside the class constructor and passes true as its 'initialize_thread' parameter meaning an intent to associate the calling thread with this connection. Before this change, the construction was done inside the very first call to 'do_build()'. The regular ("new instance per request" ) implementation of the 'basic_sql_context_builder', 'default_sql_context_builder', now passes false as the 'initialize_thread' parameter (meaning no association with the thread needed). Significantly reworked 'dictionary_flusher_thread': - instead of composed 'query_cache' object it now expects its component 'query_cache_core' and 'query_builder' as constructor arguments. This allows to create an instance of the 'static_sql_context_builder' and 'query_cache' directly inside the thread function. - Instead of 'stopped_' boolean flag, we now have a state enumeration ('initial' 'initialization_failure', 'operational', 'stopped') - the implementation no longer uses 'std::conditional_variable' for awaiting timer events / termination requests. Instead, it just wakes up periodically (once per second) and checks it it needs to reload the cache. This is necessary to be able to respond to graceful termination requests like 'KILL CONNECTION' or shutdown closing sessions at shutdown. - added 'request_termination()' method used inside component 'deinit()' handler. - 'do_periodic_reload()' function now looks more lake a state machine performing different actions and making transitions to different states. - added new logic to wait for Sessin Server availability inside the 'do_periodic_reload()' function. Reworked 'thread_handler_context' class - it now uses 'mysql_command_thread' service to initialize / deinitialize the thread for MySQL. Various MTR test case that use dictionary functions updated with explicit granting necessary privileges on the dictionary table to the 'mysql.session'@'localhost' internal MySQL user. Added new 'component_masking_functions.flusher_thread_connection_reuse' MTR test case that checks that the same MySQL internal connection (created via 'mysql_command_xxx' services) can be used several times (without closing and re-opening) by the background flusher thread. Added new 'component_masking_functions.flusher_thread_immediate_restart' MTR test case that check for proper behavior during server shutdown immediately after installing the component. Added new 'wait_for_component_uninstall.inc' MTR include file which can be used to perform several attempts to 'UNINSTALL COMPONENT' until it succeeds or reaches the max number of attempts. PS-9148 feature: basic_sql_context_builder renamed to abstract_sql_context_builder https://perconadev.atlassian.net/browse/PS-9148 Added a comment describing class hierarchy. PS-9148 feature: Extended class description comments https://perconadev.atlassian.net/browse/PS-9148 PS-9148 feature: query_cache[_core] renamed to term_cache[_core] https://perconadev.atlassian.net/browse/PS-9148 Added class descriptions for 'term_cache_core' and 'term_cache'. PS-9148 feature: query_builder transformed into a singleton https://perconadev.atlassian.net/browse/PS-9148 Removed all the boilerplate code connected with passing 'query_builder' trhough class hierarchy. 'query_builder_ptr' changed from 'std::shared_ptr' to 'std::unique_ptr'. 'term_cache_ptr' changed from 'std::shared_ptr' to 'std::unique_ptr'. PS-9148 feature: Added missing debug_sync facility reset to the MTR test cases https://perconadev.atlassian.net/browse/PS-9148 PS-9148 feature: Extended component_masking_functions.dictionary_operations https://perconadev.atlassian.net/browse/PS-9148 'component_masking_functions.dictionary_operations' extended with more checks for the case when 'mysql.session'@'localhost' system user does not have enough privileges to access 'mysql.masking_dictionaries' table. Also fixed checks for non-existing 'mysql.masking_dictionaries' table. PS-9148 feature: Fixed expected American Express card number length https://perconadev.atlassian.net/browse/PS-9148 PS-9148 feature: Added more comments about flusher thread termination https://perconadev.atlassian.net/browse/PS-9148 PS-9148 feature: Reworked bookshelf class 'bookshelf' class reworked so that internally it now holds 'std::unordered_map' of 'dictionary' objects instead of 'dictionary_ptr' objects. PS-9148 feature: Fixed compilation problem with older STLs https://perconadev.atlassian.net/browse/PS-9148 Although in recent versions of STL implementation is is OK to use 'std::unordered_map' with incomplete types, this is not true for STL coming with GCC 11 (default on Ubuntu Jammy). 'bookshelf.hpp' header now includes 'dictionary.hpp"' instead of 'dictionary_fwd.hpp"' to resolve ths issue. PS-9148 feature: Reworked flusher thread initialization https://perconadev.atlassian.net/browse/PS-9148 Reworked Dictionary Flusher thread initialization logic. We now establish internal server connection under the 'LOCK_server_shutting_down' lock and only if the Server is not shutting down ('server_shutting_down' is still false). This helps to ensure that component's 'deinit()' function (that is called with service registry locked) is not run concurrently with constructing an instance of the 'static_sql_context_builder' class (that attempts to acquire the same service registry lock). 'sql_context' class constructor now accepts 2 parameters: * initialization_registry_locking_mode (used for establishing connections) * operation_registry_locking_mode (used for closing connections) instead of a single one 'registry_locking_mode'. Because of these change component's 'deinit()' function can no longer fail. As the result, there is no need in 'wait_for_component_uninstall.inc' anymore. Changed to simple 'UNINSTALL COMPONENT'. PS-9148 feature: collation-aware lookups in Dictionary Cache https://perconadev.atlassian.net/browse/PS-9148 Reworked Dictionary Cache data structures ('dictionary' and 'bookshelf' classes): instead of storing 'utf8mb4' representations of terms in 'std::string's, we now use 'charset_string' instead. This allows to perform case insensitive accesnt aware lookups in dictionary cache following 'utf8mb4_0900_ai_ci' collation rules. Added new 'component_masking_functions.dictionary_ai_ci_lookups' MTR test case that simulates various lookups of terms that differ only in case or accent. Co-Authored-By: Oleksandr Kachan --- components/masking_functions/CMakeLists.txt | 29 + .../abstract_sql_context_builder.hpp | 70 +++ .../abstract_sql_context_builder_fwd.hpp | 30 + .../include/masking_functions/bookshelf.hpp | 63 ++ .../masking_functions/bookshelf_fwd.hpp | 29 + .../masking_functions/charset_string.hpp | 10 +- .../masking_functions/charset_string_fwd.hpp | 2 + .../command_service_tuple.hpp | 12 +- .../component_sys_variable_service_tuple.hpp | 47 ++ ...mponent_sys_variable_service_tuple_fwd.hpp | 25 + .../default_sql_context_builder.hpp | 36 ++ .../include/masking_functions/dictionary.hpp | 52 ++ .../dictionary_flusher_thread.hpp | 114 ++++ .../dictionary_flusher_thread_fwd.hpp | 30 + .../masking_functions/dictionary_fwd.hpp | 25 + .../masking_functions/query_builder.hpp | 20 +- .../masking_functions/query_builder_fwd.hpp | 29 + .../random_string_generators.hpp | 3 +- .../masking_functions/server_helpers.hpp | 36 ++ .../include/masking_functions/sql_context.hpp | 51 +- .../masking_functions/sql_context_fwd.hpp | 32 + .../static_sql_context_builder.hpp | 43 ++ .../string_service_tuple.hpp | 4 +- .../include/masking_functions/sys_vars.hpp | 34 ++ .../include/masking_functions/term_cache.hpp | 84 +++ .../masking_functions/term_cache_core.hpp | 108 ++++ .../masking_functions/term_cache_core_fwd.hpp | 29 + .../masking_functions/term_cache_fwd.hpp | 29 + .../masking_functions/src/component.cpp | 189 +++++- .../src/masking_functions/bookshelf.cpp | 78 +++ .../src/masking_functions/charset_string.cpp | 21 +- .../charset_string_operations.cpp | 1 + .../default_sql_context_builder.cpp | 30 + .../src/masking_functions/dictionary.cpp | 51 ++ .../dictionary_flusher_thread.cpp | 388 ++++++++++++ .../src/masking_functions/query_builder.cpp | 25 +- .../random_string_generators.cpp | 144 +++-- .../registration_routines.cpp | 425 +++++++------ .../src/masking_functions/server_helpers.cpp | 37 ++ .../src/masking_functions/sql_context.cpp | 196 ++++-- .../sql_escape_functions.cpp | 9 +- .../static_sql_context_builder.cpp | 37 ++ .../src/masking_functions/sys_vars.cpp | 129 ++++ .../src/masking_functions/term_cache.cpp | 56 ++ .../src/masking_functions/term_cache_core.cpp | 237 +++++++ .../have_masking_functions_component.inc | 2 +- .../r/dictionary_ai_ci_lookups.result | 43 ++ .../r/dictionary_operations.result | 577 +++++------------- .../r/flusher_thread_connection_reuse.result | 18 + .../r/flusher_thread_immediate_restart.result | 15 + .../r/flusher_thread_suspend_resume.result | 25 + .../r/global_autocommit_off.result | 2 + .../r/rpl_dictionaries_flush_interval.result | 81 +++ .../r/rpl_function_determinism_mixed.result | 2 + .../r/rpl_function_determinism_row.result | 2 + .../rpl_function_determinism_statement.result | 2 + ...naries_flush_interval_seconds_basic.result | 37 ++ .../r/sys_var_masking_database.result | 28 + .../t/check_expression.inc | 2 +- .../t/dictionary_ai_ci_lookups.test | 113 ++++ .../t/dictionary_operations.test | 232 +++++-- ...flusher_thread_connection_reuse-master.opt | 1 + .../t/flusher_thread_connection_reuse.test | 42 ++ ...lusher_thread_immediate_restart-master.opt | 1 + .../t/flusher_thread_immediate_restart.test | 23 + .../flusher_thread_suspend_resume-master.opt | 1 + .../t/flusher_thread_suspend_resume.test | 71 +++ .../t/global_autocommit_off.test | 2 + ...rpl_dictionaries_flush_interval-master.opt | 1 + .../rpl_dictionaries_flush_interval-slave.opt | 1 + .../t/rpl_dictionaries_flush_interval.test | 140 +++++ .../t/rpl_function_determinism.inc | 2 + ...ionaries_flush_interval_seconds_basic.test | 55 ++ .../t/sys_var_masking_database.test | 54 ++ 74 files changed, 3857 insertions(+), 847 deletions(-) create mode 100644 components/masking_functions/include/masking_functions/abstract_sql_context_builder.hpp create mode 100644 components/masking_functions/include/masking_functions/abstract_sql_context_builder_fwd.hpp create mode 100644 components/masking_functions/include/masking_functions/bookshelf.hpp create mode 100644 components/masking_functions/include/masking_functions/bookshelf_fwd.hpp create mode 100644 components/masking_functions/include/masking_functions/component_sys_variable_service_tuple.hpp create mode 100644 components/masking_functions/include/masking_functions/component_sys_variable_service_tuple_fwd.hpp create mode 100644 components/masking_functions/include/masking_functions/default_sql_context_builder.hpp create mode 100644 components/masking_functions/include/masking_functions/dictionary.hpp create mode 100644 components/masking_functions/include/masking_functions/dictionary_flusher_thread.hpp create mode 100644 components/masking_functions/include/masking_functions/dictionary_flusher_thread_fwd.hpp create mode 100644 components/masking_functions/include/masking_functions/dictionary_fwd.hpp create mode 100644 components/masking_functions/include/masking_functions/query_builder_fwd.hpp create mode 100644 components/masking_functions/include/masking_functions/server_helpers.hpp create mode 100644 components/masking_functions/include/masking_functions/sql_context_fwd.hpp create mode 100644 components/masking_functions/include/masking_functions/static_sql_context_builder.hpp create mode 100644 components/masking_functions/include/masking_functions/sys_vars.hpp create mode 100644 components/masking_functions/include/masking_functions/term_cache.hpp create mode 100644 components/masking_functions/include/masking_functions/term_cache_core.hpp create mode 100644 components/masking_functions/include/masking_functions/term_cache_core_fwd.hpp create mode 100644 components/masking_functions/include/masking_functions/term_cache_fwd.hpp create mode 100644 components/masking_functions/src/masking_functions/bookshelf.cpp create mode 100644 components/masking_functions/src/masking_functions/default_sql_context_builder.cpp create mode 100644 components/masking_functions/src/masking_functions/dictionary.cpp create mode 100644 components/masking_functions/src/masking_functions/dictionary_flusher_thread.cpp create mode 100644 components/masking_functions/src/masking_functions/server_helpers.cpp create mode 100644 components/masking_functions/src/masking_functions/static_sql_context_builder.cpp create mode 100644 components/masking_functions/src/masking_functions/sys_vars.cpp create mode 100644 components/masking_functions/src/masking_functions/term_cache.cpp create mode 100644 components/masking_functions/src/masking_functions/term_cache_core.cpp create mode 100644 mysql-test/suite/component_masking_functions/r/dictionary_ai_ci_lookups.result create mode 100644 mysql-test/suite/component_masking_functions/r/flusher_thread_connection_reuse.result create mode 100644 mysql-test/suite/component_masking_functions/r/flusher_thread_immediate_restart.result create mode 100644 mysql-test/suite/component_masking_functions/r/flusher_thread_suspend_resume.result create mode 100644 mysql-test/suite/component_masking_functions/r/rpl_dictionaries_flush_interval.result create mode 100644 mysql-test/suite/component_masking_functions/r/sys_var_dictionaries_flush_interval_seconds_basic.result create mode 100644 mysql-test/suite/component_masking_functions/r/sys_var_masking_database.result create mode 100644 mysql-test/suite/component_masking_functions/t/dictionary_ai_ci_lookups.test create mode 100644 mysql-test/suite/component_masking_functions/t/flusher_thread_connection_reuse-master.opt create mode 100644 mysql-test/suite/component_masking_functions/t/flusher_thread_connection_reuse.test create mode 100644 mysql-test/suite/component_masking_functions/t/flusher_thread_immediate_restart-master.opt create mode 100644 mysql-test/suite/component_masking_functions/t/flusher_thread_immediate_restart.test create mode 100644 mysql-test/suite/component_masking_functions/t/flusher_thread_suspend_resume-master.opt create mode 100644 mysql-test/suite/component_masking_functions/t/flusher_thread_suspend_resume.test create mode 100644 mysql-test/suite/component_masking_functions/t/rpl_dictionaries_flush_interval-master.opt create mode 100644 mysql-test/suite/component_masking_functions/t/rpl_dictionaries_flush_interval-slave.opt create mode 100644 mysql-test/suite/component_masking_functions/t/rpl_dictionaries_flush_interval.test create mode 100644 mysql-test/suite/component_masking_functions/t/sys_var_dictionaries_flush_interval_seconds_basic.test create mode 100644 mysql-test/suite/component_masking_functions/t/sys_var_masking_database.test diff --git a/components/masking_functions/CMakeLists.txt b/components/masking_functions/CMakeLists.txt index 97d1ca56835b..b5893e279de1 100644 --- a/components/masking_functions/CMakeLists.txt +++ b/components/masking_functions/CMakeLists.txt @@ -26,27 +26,56 @@ endif() set(DATAMASKING_SOURCES src/component.cpp + src/masking_functions/bookshelf.cpp src/masking_functions/charset_string.cpp src/masking_functions/charset_string_operations.cpp + src/masking_functions/default_sql_context_builder.cpp + src/masking_functions/dictionary.cpp + src/masking_functions/dictionary_flusher_thread.cpp src/masking_functions/query_builder.cpp + src/masking_functions/term_cache.cpp + src/masking_functions/term_cache_core.cpp src/masking_functions/random_string_generators.cpp src/masking_functions/registration_routines.cpp + src/masking_functions/server_helpers.cpp src/masking_functions/sql_context.cpp src/masking_functions/sql_escape_functions.cpp + src/masking_functions/static_sql_context_builder.cpp + src/masking_functions/sys_vars.cpp + include/masking_functions/abstract_sql_context_builder_fwd.hpp + include/masking_functions/abstract_sql_context_builder.hpp + include/masking_functions/bookshelf_fwd.hpp + include/masking_functions/bookshelf.hpp include/masking_functions/charset_string_fwd.hpp include/masking_functions/charset_string.hpp include/masking_functions/charset_string_operations.hpp include/masking_functions/command_service_tuple_fwd.hpp include/masking_functions/command_service_tuple.hpp + include/masking_functions/component_sys_variable_service_tuple_fwd.hpp + include/masking_functions/component_sys_variable_service_tuple.hpp + include/masking_functions/default_sql_context_builder.hpp + include/masking_functions/dictionary_fwd.hpp + include/masking_functions/dictionary.hpp + include/masking_functions/dictionary_flusher_thread_fwd.hpp + include/masking_functions/dictionary_flusher_thread.hpp include/masking_functions/primitive_singleton.hpp + include/masking_functions/query_builder_fwd.hpp include/masking_functions/query_builder.hpp + include/masking_functions/term_cache_fwd.hpp + include/masking_functions/term_cache.hpp + include/masking_functions/term_cache_core_fwd.hpp + include/masking_functions/term_cache_core.hpp include/masking_functions/random_string_generators.hpp include/masking_functions/registration_routines.hpp + include/masking_functions/server_helpers.hpp + include/masking_functions/sql_context_fwd.hpp include/masking_functions/sql_context.hpp include/masking_functions/sql_escape_functions.hpp + include/masking_functions/static_sql_context_builder.hpp include/masking_functions/string_service_tuple_fwd.hpp include/masking_functions/string_service_tuple.hpp + include/masking_functions/sys_vars.hpp ) ### Configuration ### diff --git a/components/masking_functions/include/masking_functions/abstract_sql_context_builder.hpp b/components/masking_functions/include/masking_functions/abstract_sql_context_builder.hpp new file mode 100644 index 000000000000..3050bbebacfe --- /dev/null +++ b/components/masking_functions/include/masking_functions/abstract_sql_context_builder.hpp @@ -0,0 +1,70 @@ +/* Copyright (c) 2023 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software Foundation, + 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ + +#ifndef MASKING_FUNCTIONS_ABSTRACT_SQL_CONTEXT_BUILDER_HPP +#define MASKING_FUNCTIONS_ABSTRACT_SQL_CONTEXT_BUILDER_HPP + +#include "masking_functions/abstract_sql_context_builder_fwd.hpp" // IWYU pragma: export + +#include "masking_functions/command_service_tuple_fwd.hpp" +#include "masking_functions/sql_context_fwd.hpp" + +namespace masking_functions { + +// This class is an abstract interface for an entity that is supposed to +// return ready-to-use instances of 'sql_context' class. +// This interface has two concrete implementations: +// 1. 'default_sql_context_builder' - creates an instance of 'sql_context' +// class (establishes an internal connection) every time its 'build()' +// method is called. +// 2. 'static_sql_context_builder' - creates an instance of 'sql_context' +// class (establishes an internal connection) only once in the constructor +// and returns a reference of this shared instance every time 'build()' +// method is called. +// +// abstract_sql_context_builder +// ^ ^ +// | | +// default_sql_context_builder static_sql_context_builder +class abstract_sql_context_builder { + public: + abstract_sql_context_builder(const abstract_sql_context_builder &) = delete; + abstract_sql_context_builder &operator=( + const abstract_sql_context_builder &) = delete; + abstract_sql_context_builder(abstract_sql_context_builder &&) = delete; + abstract_sql_context_builder &operator=(abstract_sql_context_builder &&) = + delete; + + virtual ~abstract_sql_context_builder() = default; + + sql_context_ptr build() const { return do_build(); } + + protected: + explicit abstract_sql_context_builder(const command_service_tuple &services) + : services_{&services} {} + + const command_service_tuple &get_services() const noexcept { + return *services_; + } + + private: + const command_service_tuple *services_; + + virtual sql_context_ptr do_build() const = 0; +}; + +} // namespace masking_functions + +#endif // MASKING_FUNCTIONS_ABSTRACT_SQL_CONTEXT_BUILDER_HPP diff --git a/components/masking_functions/include/masking_functions/abstract_sql_context_builder_fwd.hpp b/components/masking_functions/include/masking_functions/abstract_sql_context_builder_fwd.hpp new file mode 100644 index 000000000000..4d519b3d3556 --- /dev/null +++ b/components/masking_functions/include/masking_functions/abstract_sql_context_builder_fwd.hpp @@ -0,0 +1,30 @@ +/* Copyright (c) 2023 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software Foundation, + 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ + +#ifndef MASKING_FUNCTIONS_ABSTRACT_SQL_CONTEXT_BUILDER_FWD_HPP +#define MASKING_FUNCTIONS_ABSTRACT_SQL_CONTEXT_BUILDER_FWD_HPP + +#include + +namespace masking_functions { + +class abstract_sql_context_builder; + +using abstract_sql_context_builder_ptr = + std::shared_ptr; + +} // namespace masking_functions + +#endif // MASKING_FUNCTIONS_ABSTRACT_SQL_CONTEXT_BUILDER_FWD_HPP diff --git a/components/masking_functions/include/masking_functions/bookshelf.hpp b/components/masking_functions/include/masking_functions/bookshelf.hpp new file mode 100644 index 000000000000..6a144a73b695 --- /dev/null +++ b/components/masking_functions/include/masking_functions/bookshelf.hpp @@ -0,0 +1,63 @@ +/* Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software Foundation, + 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ + +#ifndef MASKING_FUNCTIONS_BOOKSHELF_HPP +#define MASKING_FUNCTIONS_BOOKSHELF_HPP + +#include "masking_functions/bookshelf_fwd.hpp" // IWYU pragma: export + +#include + +#include "masking_functions/charset_string.hpp" +#include "masking_functions/dictionary.hpp" + +namespace masking_functions { + +// 'bookshelf' is a collection of 'dictionary' class instances used as a +// basic in-memory data structure for caching dictionary terms grouped +// by dictionary. It operates on (dictionary_name, term)-pairs. +class bookshelf { + public: + bool contains(const charset_string &dictionary_name, + const charset_string &term) const noexcept; + // returns empty charset_string if no such dictionary exist + const charset_string &get_random( + const charset_string &dictionary_name) const noexcept; + // returns true if there was at least one term in the 'dictionary_name' + // dictionary + // returns false if there was not a single term that belongs to the + // 'dictionary_name' dictionary + bool remove(const charset_string &dictionary_name) noexcept; + // returns true if the term has been successfully removed from the + // 'dictionary_name' dictionary + // returns false if the term was not present in the 'dictionary_name' + // dictionary + bool remove(const charset_string &dictionary_name, + const charset_string &term) noexcept; + // returns true if the term has been successfully inserted into the + // 'dictionary_name' dictionary + // returns false if the term is alreaddy in the 'dictionary_name' + // dictionary + bool insert(const charset_string &dictionary_name, + const charset_string &term); + + private: + using dictionary_container = std::map; + dictionary_container dictionaries_; +}; + +} // namespace masking_functions + +#endif // MASKING_FUNCTIONS_BOOKSHELF_HPP diff --git a/components/masking_functions/include/masking_functions/bookshelf_fwd.hpp b/components/masking_functions/include/masking_functions/bookshelf_fwd.hpp new file mode 100644 index 000000000000..ea5982d7c6d2 --- /dev/null +++ b/components/masking_functions/include/masking_functions/bookshelf_fwd.hpp @@ -0,0 +1,29 @@ +/* Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software Foundation, + 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ + +#ifndef MASKING_FUNCTIONS_BOOKSHELF_FWD_HPP +#define MASKING_FUNCTIONS_BOOKSHELF_FWD_HPP + +#include + +namespace masking_functions { + +class bookshelf; + +using bookshelf_ptr = std::unique_ptr; + +} // namespace masking_functions + +#endif // MASKING_FUNCTIONS_BOOKSHELF_HPP diff --git a/components/masking_functions/include/masking_functions/charset_string.hpp b/components/masking_functions/include/masking_functions/charset_string.hpp index 585684e21192..8c1fa2e6d931 100644 --- a/components/masking_functions/include/masking_functions/charset_string.hpp +++ b/components/masking_functions/include/masking_functions/charset_string.hpp @@ -16,14 +16,14 @@ #ifndef MASKING_FUNCTIONS_CHARSET_STRING_HPP #define MASKING_FUNCTIONS_CHARSET_STRING_HPP +#include "masking_functions/charset_string_fwd.hpp" // IWYU pragma: export + #include #include #include #include #include -#include "masking_functions/charset_string_fwd.hpp" - #include "masking_functions/string_service_tuple_fwd.hpp" namespace masking_functions { @@ -89,6 +89,8 @@ class charset_string { ~charset_string() = default; + bool is_default_constructed() const noexcept { return !impl_; } + const string_service_tuple &get_services() const noexcept { return *impl_.get_deleter().services; } @@ -130,8 +132,10 @@ class charset_string { private: struct deleter { - void operator()(void *ptr) const noexcept; + // NOLINTNEXTLINE(misc-non-private-member-variables-in-classes) const string_service_tuple *services; + + void operator()(void *ptr) const noexcept; }; using impl_type = std::unique_ptr; impl_type impl_; diff --git a/components/masking_functions/include/masking_functions/charset_string_fwd.hpp b/components/masking_functions/include/masking_functions/charset_string_fwd.hpp index aa31a97ec76d..cdb3879d46cd 100644 --- a/components/masking_functions/include/masking_functions/charset_string_fwd.hpp +++ b/components/masking_functions/include/masking_functions/charset_string_fwd.hpp @@ -16,6 +16,8 @@ #ifndef MASKING_FUNCTIONS_CHARSET_STRING_FWD_HPP #define MASKING_FUNCTIONS_CHARSET_STRING_FWD_HPP +#include + namespace masking_functions { class charset_string; diff --git a/components/masking_functions/include/masking_functions/command_service_tuple.hpp b/components/masking_functions/include/masking_functions/command_service_tuple.hpp index d465ba2eded6..7149d036b7ed 100644 --- a/components/masking_functions/include/masking_functions/command_service_tuple.hpp +++ b/components/masking_functions/include/masking_functions/command_service_tuple.hpp @@ -16,12 +16,12 @@ #ifndef MASKING_FUNCTIONS_COMMAND_SERVICE_TUPLE_HPP #define MASKING_FUNCTIONS_COMMAND_SERVICE_TUPLE_HPP +#include "masking_functions/command_service_tuple_fwd.hpp" // IWYU pragma: export + #include #include -#include "masking_functions/command_service_tuple_fwd.hpp" - namespace masking_functions { // A set of MySQL query services required to perform a simple query @@ -35,16 +35,22 @@ namespace masking_functions { // mysql_command_query{ // mysql_service_mysql_command_query, // mysql_service_mysql_command_query_result, +// mysql_service_mysql_command_field_info, // mysql_service_mysql_command_options, -// mysql_service_mysql_command_factory +// mysql_service_mysql_command_factory, +// mysql_service_mysql_command_error_info, +// mysql_service_mysql_command_thread // }; // ... // sql_context ctx{primitive_singleton::instance()}; struct command_service_tuple { SERVICE_TYPE(mysql_command_query) * query; SERVICE_TYPE(mysql_command_query_result) * query_result; + SERVICE_TYPE(mysql_command_field_info) * field_info; SERVICE_TYPE(mysql_command_options) * options; SERVICE_TYPE(mysql_command_factory) * factory; + SERVICE_TYPE(mysql_command_error_info) * error_info; + SERVICE_TYPE(mysql_command_thread) * thread; }; } // namespace masking_functions diff --git a/components/masking_functions/include/masking_functions/component_sys_variable_service_tuple.hpp b/components/masking_functions/include/masking_functions/component_sys_variable_service_tuple.hpp new file mode 100644 index 000000000000..807b8ab484ca --- /dev/null +++ b/components/masking_functions/include/masking_functions/component_sys_variable_service_tuple.hpp @@ -0,0 +1,47 @@ +/* Copyright (c) 2023 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software Foundation, + 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ + +#ifndef MASKING_FUNCTIONS_COMPONENT_SYS_VARIABLE_SERVICE_TUPLE_HPP +#define MASKING_FUNCTIONS_COMPONENT_SYS_VARIABLE_SERVICE_TUPLE_HPP + +#include + +#include + +#include "masking_functions/component_sys_variable_service_tuple_fwd.hpp" // IWYU pragma: export + +namespace masking_functions { + +// A set of MySQL services required to perform system variable +// registration / unregistration. +// It is recommended to be used in a combination with the +// 'primitive_singleton' class template. +// +// primitive_singleton::instance() = +// component_sys_variable_service_tuple{ +// component_sys_variable_register, +// component_sys_variable_unregister +// }; +// ... +// sql_context +// ctx{primitive_singleton::instance()}; +struct component_sys_variable_service_tuple { + SERVICE_TYPE(component_sys_variable_register) * registrator; + SERVICE_TYPE(component_sys_variable_unregister) * unregistrator; +}; + +} // namespace masking_functions + +#endif // MASKING_FUNCTIONS_COMPONENT_SYS_VARIABLE_SERVICE_TUPLE_HPP diff --git a/components/masking_functions/include/masking_functions/component_sys_variable_service_tuple_fwd.hpp b/components/masking_functions/include/masking_functions/component_sys_variable_service_tuple_fwd.hpp new file mode 100644 index 000000000000..4bf98031e9d8 --- /dev/null +++ b/components/masking_functions/include/masking_functions/component_sys_variable_service_tuple_fwd.hpp @@ -0,0 +1,25 @@ +/* Copyright (c) 2023 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software Foundation, + 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ + +#ifndef MASKING_FUNCTIONS_COMPONENT_SYS_VARIABLE_SERVICE_TUPLE_FWD_HPP +#define MASKING_FUNCTIONS_COMPONENT_SYS_VARIABLE_SERVICE_TUPLE_FWD_HPP + +namespace masking_functions { + +struct component_sys_variable_service_tuple; + +} // namespace masking_functions + +#endif // MASKING_FUNCTIONS_COMPONENT_SYS_VARIABLE_SERVICE_TUPLE_FWD_HPP diff --git a/components/masking_functions/include/masking_functions/default_sql_context_builder.hpp b/components/masking_functions/include/masking_functions/default_sql_context_builder.hpp new file mode 100644 index 000000000000..b0bd804ddf8d --- /dev/null +++ b/components/masking_functions/include/masking_functions/default_sql_context_builder.hpp @@ -0,0 +1,36 @@ +/* Copyright (c) 2023 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software Foundation, + 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ + +#ifndef MASKING_FUNCTIONS_DEFAULT_SQL_CONTEXT_BUILDER_HPP +#define MASKING_FUNCTIONS_DEFAULT_SQL_CONTEXT_BUILDER_HPP + +#include "masking_functions/abstract_sql_context_builder.hpp" // IWYU pragma: export + +#include "masking_functions/command_service_tuple_fwd.hpp" + +namespace masking_functions { + +class default_sql_context_builder : public abstract_sql_context_builder { + public: + explicit default_sql_context_builder(const command_service_tuple &services) + : abstract_sql_context_builder{services} {} + + private: + sql_context_ptr do_build() const override; +}; + +} // namespace masking_functions + +#endif // MASKING_FUNCTIONS_DEFAULT_SQL_CONTEXT_BUILDER_HPP diff --git a/components/masking_functions/include/masking_functions/dictionary.hpp b/components/masking_functions/include/masking_functions/dictionary.hpp new file mode 100644 index 000000000000..b102168bbf9c --- /dev/null +++ b/components/masking_functions/include/masking_functions/dictionary.hpp @@ -0,0 +1,52 @@ +/* Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software Foundation, + 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ + +#ifndef MASKING_FUNCTIONS_DICTIONARY_HPP +#define MASKING_FUNCTIONS_DICTIONARY_HPP + +#include "masking_functions/dictionary_fwd.hpp" // IWYU pragma: export + +#include + +#include "masking_functions/charset_string.hpp" + +namespace masking_functions { + +// 'dictionary' is a collection of terms used as a basic in-memory data +// structure for caching dictionary terms within a single dictionary. +class dictionary { + public: + static const charset_string shared_empty; + + bool is_empty() const noexcept { return terms_.empty(); } + + bool contains(const charset_string &term) const noexcept; + // returns empty charset_string if the dictionary is empty + const charset_string &get_random() const noexcept; + // returns true if the term has been successfully inserted + // returns false if the term is alreaddy in the dictionary + bool insert(const charset_string &term); + // returns true if the term has been successfully removed + // returns false if the term was not present in the dictionary + bool remove(const charset_string &term) noexcept; + + private: + using term_container = std::set; + term_container terms_; +}; + +} // namespace masking_functions + +#endif // MASKING_FUNCTIONS_DICTIONARY_HPP diff --git a/components/masking_functions/include/masking_functions/dictionary_flusher_thread.hpp b/components/masking_functions/include/masking_functions/dictionary_flusher_thread.hpp new file mode 100644 index 000000000000..cdf7734dbceb --- /dev/null +++ b/components/masking_functions/include/masking_functions/dictionary_flusher_thread.hpp @@ -0,0 +1,114 @@ +/* Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software Foundation, + 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ + +#ifndef MASKING_FUNCTIONS_DICTIONARY_FLUSHER_THREAD_HPP +#define MASKING_FUNCTIONS_DICTIONARY_FLUSHER_THREAD_HPP + +#include "masking_functions/dictionary_flusher_thread_fwd.hpp" // IWYU pragma: export + +#include +#include +#include + +#include "masking_functions/term_cache_core_fwd.hpp" + +namespace masking_functions { + +// Facts that impacted 'dictionary_flusher_thread' design. +// - 'mysql_command_xxx' services cannot be used inside component 'init()' / +// 'deinit()' handlers as that may mess up with the 'THD' object of the +// connection that executes 'INSTALL COMPONENT' / 'UNINSTALL COMPONENT'. +// Therefore, 'INSTALL COMPONENT' cannot immediately reload dictionary +// cache. It can only spawn a background thread that will do this later. +// The cache will be loaded either during the very first call to one of the +// dictionary-related UDFs or by the flusher thread, whichever happens +// first. +// +// - MySQL internal connection (an instance of the 'sql_context' class, +// which in turn uses 'mysql_command_xxx' services) must not be created +// in the component 'init()' handler but inside the background thread. +// The main reason for this is that it needs to have its own THD +// object associated with it. Internally it is done by seting +// 'MYSQL_COMMAND_LOCAL_THD_HANDLE' option for the internal connection. +// +// - MySQL Service registry is locked inside component 'init()' / 'deinit()' +// handlers. In other words, we cannot instruct component 'init()' handler +// to wait for background thread to initiate the connection as this will +// result in a deadlock. +// +// - Similarly, we cannot instruct component 'deinit()' handler to wait for +// internal connection to be closed using regular means. However, if we +// set the 'MYSQL_NO_LOCK_REGISTRY' for this internal connection, it will +// be closed without trying to lock MySQL Service registry (which is +// already locked by the 'UNINSTALL COMPONENT' logic). +// +// - During startup when server installs components that are marked for +// loading in the Data Dictionary, Session Server is not yet available +// ('srv_session_server_is_available()' returns false) and +// 'mysql_command_xxx' cannot be used immediately. Therefore, the +// first step the background thread needs to do is to wait until the +// Session Server becomes available (until +// 'srv_session_server_is_available()' returns true) and only then +// initiate an internal connection. +// +// - During shutdown MySQL server before uninstalling components tries +// to gracefully close all remaining sessions (those registered in the +// session manager). Our background thread is also in this list as it +// sets 'MYSQL_COMMAND_LOCAL_THD_HANDLE' option for the internal +// connection. Therefore, our background thread needs to respond to +// 'KILL CONNECTION' statement (to setting 'thd->is_killed') because +// otherwise the thread will be killed by force (via 'ptheread_cancel()' +// or similar) which most probably will result in server crash. +// +// - In rare cases when 'UNINSTALL COMPONENT' is called immediately after +// 'INSTALL COMPONENT' (and when background thread has been spawned but +// not yet initialized the connection), the only safe way to avoid +// deadlocks is to let 'UNINSTALL COMPONENT' ('deinit()' handler) to fail +// earlier (without waiting for the background thread to join) by just +// requesting it to stop later (by setting the state to 'stopped'). +// Performing several attempts to 'UNINSTALL COMPONENT' should eventually +// succeed. +class dictionary_flusher_thread { + public: + dictionary_flusher_thread(const term_cache_core_ptr &cache_core, + std::uint64_t flush_interval_seconds); + dictionary_flusher_thread(const dictionary_flusher_thread &other) = delete; + dictionary_flusher_thread(dictionary_flusher_thread &&other) = delete; + dictionary_flusher_thread &operator=(const dictionary_flusher_thread &other) = + delete; + dictionary_flusher_thread &operator=(dictionary_flusher_thread &&other) = + delete; + ~dictionary_flusher_thread(); + + private: + term_cache_core_ptr cache_core_; + std::uint64_t flush_interval_seconds_; + + enum class state_type : std::uint8_t; + using atomic_state_type = std::atomic; + atomic_state_type state_; + + struct jthread_deleter { + void operator()(void *ptr) const noexcept; + }; + using jthread_ptr = std::unique_ptr; + jthread_ptr thread_impl_; + + void do_periodic_reload(); +}; + +} // namespace masking_functions + +#endif // MASKING_FUNCTIONS_DICTIONARY_FLUSHER_THREAD_HPP diff --git a/components/masking_functions/include/masking_functions/dictionary_flusher_thread_fwd.hpp b/components/masking_functions/include/masking_functions/dictionary_flusher_thread_fwd.hpp new file mode 100644 index 000000000000..d27b5d54eaac --- /dev/null +++ b/components/masking_functions/include/masking_functions/dictionary_flusher_thread_fwd.hpp @@ -0,0 +1,30 @@ +/* Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software Foundation, + 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ + +#ifndef MASKING_FUNCTIONS_DICTIONARY_FLUSHER_THREAD_FWD_HPP +#define MASKING_FUNCTIONS_DICTIONARY_FLUSHER_THREAD_FWD_HPP + +#include + +namespace masking_functions { + +class dictionary_flusher_thread; + +using dictionary_flusher_thread_ptr = + std::unique_ptr; + +} // namespace masking_functions + +#endif // MASKING_FUNCTIONS_DICTIONARY_FLUSHER_THREAD_FWD_HPP diff --git a/components/masking_functions/include/masking_functions/dictionary_fwd.hpp b/components/masking_functions/include/masking_functions/dictionary_fwd.hpp new file mode 100644 index 000000000000..c45044682da3 --- /dev/null +++ b/components/masking_functions/include/masking_functions/dictionary_fwd.hpp @@ -0,0 +1,25 @@ +/* Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software Foundation, + 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ + +#ifndef MASKING_FUNCTIONS_DICTIONARY_FWD_HPP +#define MASKING_FUNCTIONS_DICTIONARY_FWD_HPP + +namespace masking_functions { + +class dictionary; + +} // namespace masking_functions + +#endif // MASKING_FUNCTIONS_DICTIONARY_FWD_HPP diff --git a/components/masking_functions/include/masking_functions/query_builder.hpp b/components/masking_functions/include/masking_functions/query_builder.hpp index 3ecfb84042b6..ab5ebfbc2b52 100644 --- a/components/masking_functions/include/masking_functions/query_builder.hpp +++ b/components/masking_functions/include/masking_functions/query_builder.hpp @@ -16,6 +16,8 @@ #ifndef MASKING_FUNCTIONS_QUERY_BUILDER_HPP #define MASKING_FUNCTIONS_QUERY_BUILDER_HPP +#include "masking_functions/query_builder_fwd.hpp" // IWYU pragma: export + #include #include @@ -29,14 +31,13 @@ class query_builder { public: static constexpr std::string_view default_result_character_set = "utf8mb4"; - static constexpr std::string_view default_database_name = "mysql"; static constexpr std::string_view default_table_name = "masking_dictionaries"; static constexpr std::string_view default_dictionary_field_name = "Dictionary"; static constexpr std::string_view default_term_field_name = "Term"; - query_builder( - std::string_view database_name = default_database_name, + explicit query_builder( + std::string_view database_name, std::string_view table_name = default_table_name, std::string_view dictionary_field_name = default_dictionary_field_name, std::string_view term_field_name = default_term_field_name) @@ -56,14 +57,7 @@ class query_builder { return term_field_name_; } - std::string select_random_term_for_dictionary( - const charset_string &dictionary_name) const { - return select_term_for_dictionary_internal(dictionary_name, nullptr); - } - std::string check_term_presence_in_dictionary( - const charset_string &dictionary_name, const charset_string &term) const { - return select_term_for_dictionary_internal(dictionary_name, &term); - } + std::string select_all_from_dictionary() const; std::string insert_ignore_record(const charset_string &dictionary_name, const charset_string &term) const; @@ -85,10 +79,6 @@ class query_builder { std::string dictionary_field_name_; std::string term_field_name_; - std::string select_term_for_dictionary_internal( - const charset_string &dictionary_name, - const charset_string *opt_term) const; - std::string delete_for_dictionary_and_opt_term_internal( const charset_string &dictionary_name, const charset_string *opt_term) const; diff --git a/components/masking_functions/include/masking_functions/query_builder_fwd.hpp b/components/masking_functions/include/masking_functions/query_builder_fwd.hpp new file mode 100644 index 000000000000..4912dfd080d0 --- /dev/null +++ b/components/masking_functions/include/masking_functions/query_builder_fwd.hpp @@ -0,0 +1,29 @@ +/* Copyright (c) 2023 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software Foundation, + 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ + +#ifndef MASKING_FUNCTIONS_QUERY_BUILDER_FWD_HPP +#define MASKING_FUNCTIONS_QUERY_BUILDER_FWD_HPP + +#include + +namespace masking_functions { + +class query_builder; + +using query_builder_ptr = std::unique_ptr; + +} // namespace masking_functions + +#endif // MASKING_FUNCTIONS_QUERY_BUILDER_FWD_HPP diff --git a/components/masking_functions/include/masking_functions/random_string_generators.hpp b/components/masking_functions/include/masking_functions/random_string_generators.hpp index 03df010f424f..fb6770ccc44f 100644 --- a/components/masking_functions/include/masking_functions/random_string_generators.hpp +++ b/components/masking_functions/include/masking_functions/random_string_generators.hpp @@ -18,6 +18,7 @@ #define MASKING_FUNCTIONS_RANDOM_STRING_GENERATORS_HPP #include +#include #include #include @@ -28,7 +29,7 @@ namespace masking_functions { // An auxiliary enum used to specify desired character type in the // 'random_character_class_string' function. -enum class character_class { +enum class character_class : std::uint8_t { lower_alpha, // [a-z] upper_alpha, // [A-Z] numeric, // [0-9] diff --git a/components/masking_functions/include/masking_functions/server_helpers.hpp b/components/masking_functions/include/masking_functions/server_helpers.hpp new file mode 100644 index 000000000000..40dfc0bf95a5 --- /dev/null +++ b/components/masking_functions/include/masking_functions/server_helpers.hpp @@ -0,0 +1,36 @@ +/* Copyright (c) 2023 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software Foundation, + 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ + +#ifndef MASKING_FUNCTIONS_SERVER_HELPERS_HPP +#define MASKING_FUNCTIONS_SERVER_HELPERS_HPP + +#include + +namespace masking_functions { + +using lock_protected_function = std::function; + +// this function takes a read lock on 'LOCK_server_shutting_down' and +// executes 'func' only if the Server is not in SHUTDOWN mode +// ('server_shutting_down' is false) +// returns false if the Server is in SHUTDOWN mode +// returns true if the Server is not in SHUTDOWN mode and func executed +// without exceptions +// this function may throw if 'func' throws +bool execute_under_lock_if_not_in_shutdown(const lock_protected_function &func); + +} // namespace masking_functions + +#endif // MASKING_FUNCTIONS_SERVER_HELPERS_HPP diff --git a/components/masking_functions/include/masking_functions/sql_context.hpp b/components/masking_functions/include/masking_functions/sql_context.hpp index 3a7aee933d30..4dd08c5abe59 100644 --- a/components/masking_functions/include/masking_functions/sql_context.hpp +++ b/components/masking_functions/include/masking_functions/sql_context.hpp @@ -16,7 +16,12 @@ #ifndef MASKING_FUNCTIONS_SQL_CONTEXT_HPP #define MASKING_FUNCTIONS_SQL_CONTEXT_HPP -#include +#include "masking_functions/sql_context_fwd.hpp" // IWYU pragma: export + +#include +#include +#include +#include #include #include @@ -30,9 +35,17 @@ namespace masking_functions { // construction. class sql_context { public: - using optional_string = std::optional; + template + using field_value_container = std::array; + + template + using row_callback = + std::function &)>; - explicit sql_context(const command_service_tuple &services); + sql_context(const command_service_tuple &services, + sql_context_registry_access initialization_registry_locking_mode, + sql_context_registry_access operation_registry_locking_mode, + bool initialize_thread); sql_context(sql_context const &) = delete; sql_context(sql_context &&) noexcept = default; @@ -42,23 +55,45 @@ class sql_context { ~sql_context() = default; + void reset(); + const command_service_tuple &get_services() const noexcept { return *impl_.get_deleter().services; } - // Executes a query where we either expect a single result (one row one - // column), or nothing - optional_string query_single_value(std::string_view query); + template + void execute_select(std::string_view query, + const row_callback &callback) { + execute_select_internal( + query, NumberOfFields, + [&callback](char **field_values, std::size_t *lengths) { + field_value_container wrapped_field_values; + std::transform(field_values, field_values + NumberOfFields, lengths, + std::begin(wrapped_field_values), + [](char *str, std::size_t len) { + return std::string_view{str, len}; + }); + callback(wrapped_field_values); + }); + } - bool execute(std::string_view query); + bool execute_dml(std::string_view query); private: struct deleter { - void operator()(void *ptr) const noexcept; + // NOLINTNEXTLINE(misc-non-private-member-variables-in-classes) const command_service_tuple *services; + + void operator()(void *ptr) const noexcept; }; using impl_type = std::unique_ptr; impl_type impl_; + + using row_internal_callback = std::function; + void execute_select_internal(std::string_view query, + std::size_t number_of_fields, + const row_internal_callback &callback); + [[noreturn]] void raise_with_error_message(std::string_view prefix); }; } // namespace masking_functions diff --git a/components/masking_functions/include/masking_functions/sql_context_fwd.hpp b/components/masking_functions/include/masking_functions/sql_context_fwd.hpp new file mode 100644 index 000000000000..817cc558b4e5 --- /dev/null +++ b/components/masking_functions/include/masking_functions/sql_context_fwd.hpp @@ -0,0 +1,32 @@ +/* Copyright (c) 2023 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software Foundation, + 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ + +#ifndef MASKING_FUNCTIONS_SQL_CONTEXT_FWD_HPP +#define MASKING_FUNCTIONS_SQL_CONTEXT_FWD_HPP + +#include +#include + +namespace masking_functions { + +class sql_context; + +using sql_context_ptr = std::shared_ptr; + +enum class sql_context_registry_access : std::uint8_t { non_locking, locking }; + +} // namespace masking_functions + +#endif // MASKING_FUNCTIONS_SQL_CONTEXT_FWD_HPP diff --git a/components/masking_functions/include/masking_functions/static_sql_context_builder.hpp b/components/masking_functions/include/masking_functions/static_sql_context_builder.hpp new file mode 100644 index 000000000000..6f5f823ec12c --- /dev/null +++ b/components/masking_functions/include/masking_functions/static_sql_context_builder.hpp @@ -0,0 +1,43 @@ +/* Copyright (c) 2023 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software Foundation, + 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ + +#ifndef MASKING_FUNCTIONS_STATIC_SQL_CONTEXT_BUILDER_HPP +#define MASKING_FUNCTIONS_STATIC_SQL_CONTEXT_BUILDER_HPP + +#include "masking_functions/abstract_sql_context_builder.hpp" // IWYU pragma: export + +#include "masking_functions/command_service_tuple_fwd.hpp" + +namespace masking_functions { + +class static_sql_context_builder : public abstract_sql_context_builder { + public: + explicit static_sql_context_builder(const command_service_tuple &services); + static_sql_context_builder(const static_sql_context_builder &) = delete; + static_sql_context_builder &operator=(const static_sql_context_builder &) = + delete; + static_sql_context_builder(static_sql_context_builder &&) = delete; + static_sql_context_builder &operator=(static_sql_context_builder &&) = delete; + ~static_sql_context_builder() override; + + private: + sql_context_ptr static_instance_; + + sql_context_ptr do_build() const override; +}; + +} // namespace masking_functions + +#endif // MASKING_FUNCTIONS_STATIC_SQL_CONTEXT_BUILDER_HPP diff --git a/components/masking_functions/include/masking_functions/string_service_tuple.hpp b/components/masking_functions/include/masking_functions/string_service_tuple.hpp index e972a10846c4..d9fe5ac81704 100644 --- a/components/masking_functions/include/masking_functions/string_service_tuple.hpp +++ b/components/masking_functions/include/masking_functions/string_service_tuple.hpp @@ -16,11 +16,11 @@ #ifndef MASKING_FUNCTIONS_STRING_SERVICE_TUPLE_HPP #define MASKING_FUNCTIONS_STRING_SERVICE_TUPLE_HPP +#include "masking_functions/string_service_tuple_fwd.hpp" // IWYU pragma: export + #include #include -#include "masking_functions/string_service_tuple_fwd.hpp" - namespace masking_functions { // A set of MySQL string manipulation services required to perform character diff --git a/components/masking_functions/include/masking_functions/sys_vars.hpp b/components/masking_functions/include/masking_functions/sys_vars.hpp new file mode 100644 index 000000000000..767eccf3a9d4 --- /dev/null +++ b/components/masking_functions/include/masking_functions/sys_vars.hpp @@ -0,0 +1,34 @@ +/* Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software Foundation, + 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ + +#ifndef MASKING_FUNCTIONS_SYS_VARS_HPP +#define MASKING_FUNCTIONS_SYS_VARS_HPP + +#include +#include +#include + +namespace masking_functions { + +std::string_view get_dict_database_name() noexcept; +std::uint64_t get_flush_interval_seconds() noexcept; + +bool register_sys_vars(); +bool unregister_sys_vars(); +bool check_sys_vars(std::string &error_message); + +} // namespace masking_functions + +#endif // MASKING_FUNCTIONS_SYS_VARS_HPP diff --git a/components/masking_functions/include/masking_functions/term_cache.hpp b/components/masking_functions/include/masking_functions/term_cache.hpp new file mode 100644 index 000000000000..dd646228c4fb --- /dev/null +++ b/components/masking_functions/include/masking_functions/term_cache.hpp @@ -0,0 +1,84 @@ +/* Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software Foundation, + 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ + +#ifndef MASKING_FUNCTIONS_TERM_CACHE_HPP +#define MASKING_FUNCTIONS_TERM_CACHE_HPP + +#include "masking_functions/term_cache_fwd.hpp" // IWYU pragma: export + +#include "masking_functions/abstract_sql_context_builder_fwd.hpp" +#include "masking_functions/charset_string_fwd.hpp" +#include "masking_functions/term_cache_core_fwd.hpp" + +namespace masking_functions { + +// 'term_cache' class allows to have a convenient wrapper over a combination +// of 'term_cache_core' and 'abstract_sql_context_builder'. +// Its 'contains()' / 'get_random()' / 'remove()' / 'insert()' methods in +// contrast to 'term_cache_core' do not have additional 'sql_ctx_builder' +// parameter of type 'abstract_sql_context_builder'. +// Basically, for the needs of data_masking component we can have a single +// instance of 'term_cache_core' and two instances of 'term_cache': +// 1. One for the dictionary-related UDFs (constructed with +// 'default_sql_context_builder' that established new connection every +// time we need to access persistent storage - 'masking_dictionaries' +// table). +// 2. One for the background reloading thread (constructed with +// 'static_sql_context_builder' that always reuses the same connection it +// established in the constructor). +class term_cache { + public: + term_cache(const term_cache_core_ptr &core, + const abstract_sql_context_builder_ptr &sql_ctx_builder); + + term_cache(const term_cache &other) = delete; + term_cache(term_cache &&other) = delete; + term_cache &operator=(const term_cache &other) = delete; + term_cache &operator=(term_cache &&other) = delete; + ~term_cache(); + + bool contains(const charset_string &dictionary_name, + const charset_string &term) const; + // returns a copy of the string to avoid race conditions + // an empty string is returned if the dictionary does not exist + charset_string get_random(const charset_string &dictionary_name) const; + // returns true if there was at least one term in the 'dictionary_name' + // dictionary + // returns false if there was not a single term that belongs to the + // 'dictionary_name' dictionary + bool remove(const charset_string &dictionary_name); + // returns true if the term has been successfully removed from the + // 'dictionary_name' dictionary + // returns false if the term was not present in the 'dictionary_name' + // dictionary + bool remove(const charset_string &dictionary_name, + const charset_string &term); + // returns true if the term has been successfully inserted into the + // 'dictionary_name' dictionary + // returns false if the term is alreaddy in the 'dictionary_name' + // dictionary + bool insert(const charset_string &dictionary_name, + const charset_string &term); + + void reload_cache(); + + private: + term_cache_core_ptr core_; + abstract_sql_context_builder_ptr sql_ctx_builder_; +}; + +} // namespace masking_functions + +#endif // MASKING_FUNCTIONS_TERM_CACHE_HPP diff --git a/components/masking_functions/include/masking_functions/term_cache_core.hpp b/components/masking_functions/include/masking_functions/term_cache_core.hpp new file mode 100644 index 000000000000..65ded8a91c82 --- /dev/null +++ b/components/masking_functions/include/masking_functions/term_cache_core.hpp @@ -0,0 +1,108 @@ +/* Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software Foundation, + 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ + +#ifndef MASKING_FUNCTIONS_TERM_CACHE_CORE_HPP +#define MASKING_FUNCTIONS_TERM_CACHE_CORE_HPP + +#include "masking_functions/term_cache_core_fwd.hpp" // IWYU pragma: export + +#include +#include +#include +#include + +#include "masking_functions/abstract_sql_context_builder_fwd.hpp" +#include "masking_functions/bookshelf_fwd.hpp" +#include "masking_functions/charset_string_fwd.hpp" +#include "masking_functions/sql_context_fwd.hpp" + +namespace masking_functions { +// 'term_cache_core' is a thread-safe class that incapsulates 'bookshelf' +// as an in-memory data structure that is used for caching dictionary terms. +// It is also aware of how to populate itself from a persisted storage +// ('masking_dictionaries' table) and how to reflect non-readonly operations +// in this persistent storage. +// Internally it uses 'abstract_sql_context_builder' as a way of establishing +// internal server session connections and executing SQL querues (constructed +// by query_builder interface). +// This class is used as a low-level abstraction ("core") of the higher-level +// user-facing class 'term_cache. +// All the methods of this class have additional 'sql_ctx_builder' parameter +// of type 'abstract_sql_context_builder' which comes handy when it is +// necessary to create different instances of the 'term_cache' class with the +// same 'term_cache_core' but with different 'abstract_sql_context_builder'. +class term_cache_core { + public: + term_cache_core(); + term_cache_core(const term_cache_core &other) = delete; + term_cache_core(term_cache_core &&other) = delete; + term_cache_core &operator=(const term_cache_core &other) = delete; + term_cache_core &operator=(term_cache_core &&other) = delete; + ~term_cache_core(); + + bool contains(const abstract_sql_context_builder &sql_ctx_builder, + const charset_string &dictionary_name, + const charset_string &term) const; + // returns a copy of the string to avoid race conditions + // an empty string is returned if the dictionary does not exist + charset_string get_random(const abstract_sql_context_builder &sql_ctx_builder, + const charset_string &dictionary_name) const; + // returns true if there was at least one term in the 'dictionary_name' + // dictionary + // returns false if there was not a single term that belongs to the + // 'dictionary_name' dictionary + bool remove(const abstract_sql_context_builder &sql_ctx_builder, + const charset_string &dictionary_name); + // returns true if the term has been successfully removed from the + // 'dictionary_name' dictionary + // returns false if the term was not present in the 'dictionary_name' + // dictionary + bool remove(const abstract_sql_context_builder &sql_ctx_builder, + const charset_string &dictionary_name, + const charset_string &term); + // returns true if the term has been successfully inserted into the + // 'dictionary_name' dictionary + // returns false if the term is alreaddy in the 'dictionary_name' + // dictionary + bool insert(const abstract_sql_context_builder &sql_ctx_builder, + const charset_string &dictionary_name, + const charset_string &term); + + void reload_cache(const abstract_sql_context_builder &sql_ctx_builder); + + private: + mutable bookshelf_ptr dict_cache_; + mutable std::shared_mutex dict_cache_mutex_; + + static bookshelf_ptr create_dict_cache_internal(sql_context &sql_ctx, + std::string &error_message); + + using shared_lock_type = std::shared_lock; + using unique_lock_type = std::unique_lock; + const bookshelf &acquire_dict_cache_shared( + const abstract_sql_context_builder &sql_ctx_builder, + sql_context_ptr &sql_ctx, shared_lock_type &read_lock, + unique_lock_type &write_lock) const; + bookshelf &acquire_dict_cache_unique( + const abstract_sql_context_builder &sql_ctx_builder, + sql_context_ptr &sql_ctx, unique_lock_type &write_lock) const; + + static const charset_string &to_utf8mb4(const charset_string &str, + charset_string &buffer); +}; + +} // namespace masking_functions + +#endif // MASKING_FUNCTIONS_TERM_CACHE_CORE_HPP diff --git a/components/masking_functions/include/masking_functions/term_cache_core_fwd.hpp b/components/masking_functions/include/masking_functions/term_cache_core_fwd.hpp new file mode 100644 index 000000000000..13c206d2698e --- /dev/null +++ b/components/masking_functions/include/masking_functions/term_cache_core_fwd.hpp @@ -0,0 +1,29 @@ +/* Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software Foundation, + 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ + +#ifndef MASKING_FUNCTIONS_TERM_CACHE_CORE_FWD_HPP +#define MASKING_FUNCTIONS_TERM_CACHE_CORE_FWD_HPP + +#include + +namespace masking_functions { + +class term_cache_core; + +using term_cache_core_ptr = std::shared_ptr; + +} // namespace masking_functions + +#endif // MASKING_FUNCTIONS_TERM_CACHE_CORE_FWD_HPP diff --git a/components/masking_functions/include/masking_functions/term_cache_fwd.hpp b/components/masking_functions/include/masking_functions/term_cache_fwd.hpp new file mode 100644 index 000000000000..21edd7fed2d2 --- /dev/null +++ b/components/masking_functions/include/masking_functions/term_cache_fwd.hpp @@ -0,0 +1,29 @@ +/* Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software Foundation, + 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ + +#ifndef MASKING_FUNCTIONS_TERM_CACHE_FWD_HPP +#define MASKING_FUNCTIONS_TERM_CACHE_FWD_HPP + +#include + +namespace masking_functions { + +class term_cache; + +using term_cache_ptr = std::unique_ptr; + +} // namespace masking_functions + +#endif // MASKING_FUNCTIONS_TERM_CACHE_FWD_HPP diff --git a/components/masking_functions/src/component.cpp b/components/masking_functions/src/component.cpp index 7650d571a899..e6decab0df5d 100644 --- a/components/masking_functions/src/component.cpp +++ b/components/masking_functions/src/component.cpp @@ -14,28 +14,50 @@ along with this program; if not, write to the Free Software Foundation, 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ +#include +#include +#include +#include +#include +#include +#include + #include +#include +#include +#include +#include + #include +#include -#include +#include // IWYU pragma: keep #include -#include -#include +#include // IWYU pragma: keep +#include // IWYU pragma: keep #include -#include -#include -#include -#include +#include // IWYU pragma: keep +#include +#include // IWYU pragma: keep +#include // IWYU pragma: keep +#include // IWYU pragma: keep #include -#include +#include "sql/debug_sync.h" // IWYU pragma: keep #include "masking_functions/command_service_tuple.hpp" +#include "masking_functions/component_sys_variable_service_tuple.hpp" +#include "masking_functions/default_sql_context_builder.hpp" +#include "masking_functions/dictionary_flusher_thread.hpp" #include "masking_functions/primitive_singleton.hpp" +#include "masking_functions/query_builder.hpp" #include "masking_functions/registration_routines.hpp" #include "masking_functions/string_service_tuple.hpp" +#include "masking_functions/sys_vars.hpp" +#include "masking_functions/term_cache.hpp" +#include "masking_functions/term_cache_core.hpp" // defined as a macro because needed both raw and stringized #define CURRENT_COMPONENT_NAME masking_functions @@ -54,8 +76,13 @@ REQUIRES_SERVICE_PLACEHOLDER(mysql_string_compare); REQUIRES_SERVICE_PLACEHOLDER(mysql_command_query); REQUIRES_SERVICE_PLACEHOLDER(mysql_command_query_result); +REQUIRES_SERVICE_PLACEHOLDER(mysql_command_field_info); REQUIRES_SERVICE_PLACEHOLDER(mysql_command_options); REQUIRES_SERVICE_PLACEHOLDER(mysql_command_factory); +REQUIRES_SERVICE_PLACEHOLDER(mysql_command_error_info); +REQUIRES_SERVICE_PLACEHOLDER(mysql_command_thread); + +REQUIRES_PSI_THREAD_SERVICE_PLACEHOLDER; REQUIRES_SERVICE_PLACEHOLDER(udf_registration); REQUIRES_SERVICE_PLACEHOLDER(dynamic_privilege_register); @@ -65,6 +92,8 @@ REQUIRES_SERVICE_PLACEHOLDER(mysql_udf_metadata); REQUIRES_SERVICE_PLACEHOLDER(mysql_current_thread_reader); REQUIRES_SERVICE_PLACEHOLDER(mysql_thd_security_context); REQUIRES_SERVICE_PLACEHOLDER(global_grants_check); +REQUIRES_SERVICE_PLACEHOLDER(component_sys_variable_register); +REQUIRES_SERVICE_PLACEHOLDER(component_sys_variable_unregister); REQUIRES_SERVICE_PLACEHOLDER(log_builtins); REQUIRES_SERVICE_PLACEHOLDER(log_builtins_string); @@ -74,17 +103,21 @@ REQUIRES_SERVICE_PLACEHOLDER(mysql_runtime_error); SERVICE_TYPE(log_builtins) * log_bi; SERVICE_TYPE(log_builtins_string) * log_bs; -static mysql_service_status_t component_init(); -static mysql_service_status_t component_deinit(); +namespace { + +mysql_service_status_t component_init(); +mysql_service_status_t component_deinit(); -static void masking_functions_my_error(int error_id, myf flags, ...) { +void masking_functions_my_error(int error_id, myf flags, ...) { va_list args; va_start(args, flags); mysql_service_mysql_runtime_error->emit(error_id, flags, args); va_end(args); } -static mysql_service_status_t component_init() { +mysql_service_status_t component_init() { + mysql_service_status_t initialization_result{0}; + log_bi = mysql_service_log_builtins; log_bs = mysql_service_log_builtins_string; @@ -108,8 +141,17 @@ static mysql_service_status_t component_init() { // TODO: convert this to designated initializers in c++20 mysql_service_mysql_command_query, mysql_service_mysql_command_query_result, + mysql_service_mysql_command_field_info, mysql_service_mysql_command_options, - mysql_service_mysql_command_factory}; + mysql_service_mysql_command_factory, + mysql_service_mysql_command_error_info, + mysql_service_mysql_command_thread}; + masking_functions::primitive_singleton< + masking_functions::component_sys_variable_service_tuple>::instance() = + masking_functions::component_sys_variable_service_tuple{ + // TODO: convert this to designated initializers in c++20 + mysql_service_component_sys_variable_register, + mysql_service_component_sys_variable_unregister}; // here we use a custom error reporting function // 'masking_functions_my_error()' based on the @@ -118,31 +160,105 @@ static mysql_service_status_t component_init() { // component mysqlpp::udf_error_reporter::instance() = &masking_functions_my_error; - if (!masking_functions::register_dynamic_privileges()) { - LogComponentErr(ERROR_LEVEL, ER_LOG_PRINTF_MSG, - "Cannot register dynamic privilege"); - component_deinit(); - return 1; - } + try { + if (!masking_functions::register_dynamic_privileges()) { + throw std::runtime_error{"Cannot register dynamic privilege"}; + } + + if (!masking_functions::register_sys_vars()) { + throw std::runtime_error{"Cannot register system variables"}; + } + + std::string check_error_message; + if (!masking_functions::check_sys_vars(check_error_message)) { + throw std::runtime_error{check_error_message}; + } + + if (!masking_functions::register_udfs()) { + throw std::runtime_error{"Cannot register UDFs"}; + } + + auto sql_query_builder{std::make_unique( + masking_functions::get_dict_database_name())}; + masking_functions::primitive_singleton< + masking_functions::query_builder_ptr>::instance() = + std::move(sql_query_builder); + + const auto &command_services = masking_functions::primitive_singleton< + masking_functions::command_service_tuple>::instance(); + + auto default_sql_ctx_builder{ + std::make_shared( + command_services)}; + + // here we create an instance of the 'term_cache_core' class that will + // be shared between 'primary_cache' (an instance of the 'term_cache' + // created with this core and 'default_sql_context_builder') and another + // instance of the 'term_cache' class created inside background thread + // handler function with this core and 'static_sql_context_builder' + auto cache_core{std::make_shared()}; + + auto primary_cache{std::make_unique( + cache_core, default_sql_ctx_builder)}; + masking_functions::primitive_singleton< + masking_functions::term_cache_ptr>::instance() = + std::move(primary_cache); + + const auto flush_interval_seconds{ + masking_functions::get_flush_interval_seconds()}; + if (flush_interval_seconds > 0U) { + auto flusher{ + std::make_unique( + cache_core, flush_interval_seconds)}; + + masking_functions::primitive_singleton< + masking_functions::dictionary_flusher_thread_ptr>::instance() = + std::move(flusher); + + // NOLINTNEXTLINE(cppcoreguidelines-avoid-do-while) + DBUG_EXECUTE_IF("enable_masking_functions_flusher_create_sync", { + MYSQL_THD extracted_thd{nullptr}; + mysql_service_mysql_current_thread_reader->get(&extracted_thd); + assert(extracted_thd != nullptr); + DEBUG_SYNC(extracted_thd, "masking_functions_after_flusher_create"); + }); + } - if (!masking_functions::register_udfs()) { - LogComponentErr(ERROR_LEVEL, ER_LOG_PRINTF_MSG, "Cannot register UDFs"); + LogComponentErr(INFORMATION_LEVEL, ER_LOG_PRINTF_MSG, + "Component successfully initialized"); + } catch (const std::exception &e) { + LogComponentErr(ERROR_LEVEL, ER_LOG_PRINTF_MSG, e.what()); component_deinit(); - return 1; + initialization_result = 1; } - - LogComponentErr(INFORMATION_LEVEL, ER_LOG_PRINTF_MSG, - "Component successfully initialized"); - return 0; + return initialization_result; } -static mysql_service_status_t component_deinit() { +mysql_service_status_t component_deinit() { int result = 0; + auto &flusher{masking_functions::primitive_singleton< + masking_functions::dictionary_flusher_thread_ptr>::instance()}; + + // the destruction of the 'flusher' object will also trigger graceful + // background thread termination ('dictionary_flusher_thread' destructor + // will set its state to 'stopped' and then will join the thread) + flusher.reset(); + + masking_functions::primitive_singleton< + masking_functions::term_cache_ptr>::instance() + .reset(); + if (!masking_functions::unregister_udfs()) { LogComponentErr(ERROR_LEVEL, ER_LOG_PRINTF_MSG, "Cannot unregister UDFs"); result = 1; } + if (!masking_functions::unregister_sys_vars()) { + LogComponentErr(ERROR_LEVEL, ER_LOG_PRINTF_MSG, + "Cannot unregister system variables"); + result = 1; + } + if (!masking_functions::unregister_dynamic_privileges()) { LogComponentErr(ERROR_LEVEL, ER_LOG_PRINTF_MSG, "Cannot unregister dynamic privilege"); @@ -158,7 +274,14 @@ static mysql_service_status_t component_deinit() { return result; } +} // anonymous namespace + // clang-format off + +// NOLINTBEGIN(cppcoreguidelines-pro-type-const-cast) +// NOLINTBEGIN(bugprone-multi-level-implicit-pointer-conversion) +// NOLINTBEGIN(bugprone-casting-through-void) +// NOLINTBEGIN(misc-use-anonymous-namespace) BEGIN_COMPONENT_PROVIDES(CURRENT_COMPONENT_NAME) END_COMPONENT_PROVIDES(); @@ -174,10 +297,15 @@ BEGIN_COMPONENT_REQUIRES(CURRENT_COMPONENT_NAME) REQUIRES_SERVICE(mysql_string_substr), REQUIRES_SERVICE(mysql_string_compare), + REQUIRES_PSI_THREAD_SERVICE, + REQUIRES_SERVICE(mysql_command_query), REQUIRES_SERVICE(mysql_command_query_result), + REQUIRES_SERVICE(mysql_command_field_info), REQUIRES_SERVICE(mysql_command_options), REQUIRES_SERVICE(mysql_command_factory), + REQUIRES_SERVICE(mysql_command_error_info), + REQUIRES_SERVICE(mysql_command_thread), REQUIRES_SERVICE(udf_registration), REQUIRES_SERVICE(dynamic_privilege_register), @@ -187,6 +315,8 @@ BEGIN_COMPONENT_REQUIRES(CURRENT_COMPONENT_NAME) REQUIRES_SERVICE(mysql_current_thread_reader), REQUIRES_SERVICE(mysql_thd_security_context), REQUIRES_SERVICE(global_grants_check), + REQUIRES_SERVICE(component_sys_variable_register), + REQUIRES_SERVICE(component_sys_variable_unregister), REQUIRES_SERVICE(log_builtins), REQUIRES_SERVICE(log_builtins_string), @@ -207,4 +337,9 @@ END_DECLARE_COMPONENT(); DECLARE_LIBRARY_COMPONENTS &COMPONENT_REF(CURRENT_COMPONENT_NAME) END_DECLARE_LIBRARY_COMPONENTS +// NOLINTEND(misc-use-anonymous-namespace) +// NOLINTEND(bugprone-casting-through-void) +// NOLINTEND(bugprone-multi-level-implicit-pointer-conversion) +// NOLINTEND(cppcoreguidelines-pro-type-const-cast) + // clang-format on diff --git a/components/masking_functions/src/masking_functions/bookshelf.cpp b/components/masking_functions/src/masking_functions/bookshelf.cpp new file mode 100644 index 000000000000..6b8fda1af834 --- /dev/null +++ b/components/masking_functions/src/masking_functions/bookshelf.cpp @@ -0,0 +1,78 @@ +/* Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software Foundation, + 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ + +#include "masking_functions/bookshelf.hpp" + +#include + +#include "masking_functions/charset_string.hpp" +#include "masking_functions/dictionary.hpp" + +namespace masking_functions { + +bool bookshelf::contains(const charset_string &dictionary_name, + const charset_string &term) const noexcept { + const auto dictionary_it{dictionaries_.find(dictionary_name)}; + if (dictionary_it == std::cend(dictionaries_)) { + return false; + } + return dictionary_it->second.contains(term); +} + +const charset_string &bookshelf::get_random( + const charset_string &dictionary_name) const noexcept { + const auto dictionary_it{dictionaries_.find(dictionary_name)}; + if (dictionary_it == std::cend(dictionaries_)) { + return dictionary::shared_empty; + } + return dictionary_it->second.get_random(); +} + +bool bookshelf::remove(const charset_string &dictionary_name) noexcept { + return dictionaries_.erase(dictionary_name) != 0U; +} + +bool bookshelf::remove(const charset_string &dictionary_name, + const charset_string &term) noexcept { + const auto dictionary_it{dictionaries_.find(dictionary_name)}; + if (dictionary_it == std::end(dictionaries_)) { + return false; + } + const auto result{dictionary_it->second.remove(term)}; + if (dictionary_it->second.is_empty()) { + dictionaries_.erase(dictionary_it); + } + return result; +} + +bool bookshelf::insert(const charset_string &dictionary_name, + const charset_string &term) { + // here we use try_emplace as a combined version of find and + // insert + const auto [dictionary_it, + inserted]{dictionaries_.try_emplace(dictionary_name)}; + if (!inserted) { + return dictionary_it->second.insert(term); + } + try { + dictionary_it->second.insert(term); + } catch (...) { + dictionaries_.erase(dictionary_it); + throw; + } + return true; +} + +} // namespace masking_functions diff --git a/components/masking_functions/src/masking_functions/charset_string.cpp b/components/masking_functions/src/masking_functions/charset_string.cpp index 869061730180..df5327576ab7 100644 --- a/components/masking_functions/src/masking_functions/charset_string.cpp +++ b/components/masking_functions/src/masking_functions/charset_string.cpp @@ -17,18 +17,21 @@ #include #include +#include #include #include #include +#include + #include "masking_functions/string_service_tuple.hpp" namespace { -my_h_string to_h_string(void *h) noexcept { - return static_cast(h); +my_h_string to_h_string(void *handle) noexcept { + return static_cast(handle); } -CHARSET_INFO_h to_cs_info_h(void *h) noexcept { - return static_cast(h); +CHARSET_INFO_h to_cs_info_h(void *handle) noexcept { + return static_cast(handle); } } // anonymous namespace @@ -56,6 +59,7 @@ charset_string::charset_string(const string_service_tuple &services, assert(local_handle != nullptr); impl_.reset(local_handle); + assert(buffer.data() != nullptr); if ((*get_services().converter->convert_from_buffer)( local_handle, buffer.data(), buffer.size(), to_cs_info_h(collation)) != 0) @@ -64,7 +68,7 @@ charset_string::charset_string(const string_service_tuple &services, std::size_t charset_string::get_size_in_characters() const noexcept { assert(impl_); - uint size_in_characters = 0; + uint size_in_characters = 0; // NOLINT(misc-include-cleaner) [[maybe_unused]] auto status = (*get_services().character_access->get_char_length)( to_h_string(impl_.get()), &size_in_characters); @@ -74,7 +78,7 @@ std::size_t charset_string::get_size_in_characters() const noexcept { std::size_t charset_string::get_size_in_bytes() const noexcept { assert(impl_); - uint size_in_bytes = 0; + uint size_in_bytes = 0; // NOLINT(misc-include-cleaner) [[maybe_unused]] auto status = (*get_services().byte_access->get_byte_length)( to_h_string(impl_.get()), &size_in_bytes); assert(status == 0); @@ -90,7 +94,7 @@ void charset_string::clear() noexcept { std::uint32_t charset_string::operator[](std::size_t index) const noexcept { assert(impl_); - ulong ch{0}; + ulong ch{0}; // NOLINT(misc-include-cleaner) [[maybe_unused]] auto status = (*get_services().character_access->get_char)( to_h_string(impl_.get()), index, &ch); assert(status == 0); @@ -181,10 +185,9 @@ int charset_string::compare(const charset_string &another) const { assert(another.impl_); int result{0}; - const auto collation = get_collation(); charset_string conversion_buffer; const charset_string &rhs = - smart_convert_to_collation(another, collation, conversion_buffer); + smart_convert_to_collation(another, get_collation(), conversion_buffer); [[maybe_unused]] auto status = (*get_services().compare->compare)( to_h_string(impl_.get()), to_h_string(rhs.impl_.get()), &result); diff --git a/components/masking_functions/src/masking_functions/charset_string_operations.cpp b/components/masking_functions/src/masking_functions/charset_string_operations.cpp index d617d7485b09..aff9cad4cf62 100644 --- a/components/masking_functions/src/masking_functions/charset_string_operations.cpp +++ b/components/masking_functions/src/masking_functions/charset_string_operations.cpp @@ -17,6 +17,7 @@ #include "masking_functions/charset_string_operations.hpp" #include +#include #include #include #include diff --git a/components/masking_functions/src/masking_functions/default_sql_context_builder.cpp b/components/masking_functions/src/masking_functions/default_sql_context_builder.cpp new file mode 100644 index 000000000000..810a62b1348e --- /dev/null +++ b/components/masking_functions/src/masking_functions/default_sql_context_builder.cpp @@ -0,0 +1,30 @@ +/* Copyright (c) 2023 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software Foundation, + 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ + +#include "masking_functions/default_sql_context_builder.hpp" + +#include + +#include "masking_functions/sql_context.hpp" + +namespace masking_functions { + +sql_context_ptr default_sql_context_builder::do_build() const { + return std::make_shared( + get_services(), sql_context_registry_access::locking, + sql_context_registry_access::locking, false); +} + +} // namespace masking_functions diff --git a/components/masking_functions/src/masking_functions/dictionary.cpp b/components/masking_functions/src/masking_functions/dictionary.cpp new file mode 100644 index 000000000000..e7d1c1190511 --- /dev/null +++ b/components/masking_functions/src/masking_functions/dictionary.cpp @@ -0,0 +1,51 @@ +/* Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software Foundation, + 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ + +#include "masking_functions/dictionary.hpp" + +#include +#include + +#include "masking_functions/charset_string.hpp" +#include "masking_functions/random_string_generators.hpp" + +namespace masking_functions { + +const charset_string dictionary::shared_empty{}; + +bool dictionary::contains(const charset_string &term) const noexcept { + // TODO: in c++20 change to terms_.contains(term) + return terms_.count(term) > 0U; +} + +const charset_string &dictionary::get_random() const noexcept { + if (terms_.empty()) { + return shared_empty; + } + + const auto random_index{random_number(0, terms_.size() - 1U)}; + return *std::next(std::begin(terms_), + static_cast(random_index)); +} + +bool dictionary::insert(const charset_string &term) { + return terms_.emplace(term).second; +} + +bool dictionary::remove(const charset_string &term) noexcept { + return terms_.erase(term) > 0U; +} + +} // namespace masking_functions diff --git a/components/masking_functions/src/masking_functions/dictionary_flusher_thread.cpp b/components/masking_functions/src/masking_functions/dictionary_flusher_thread.cpp new file mode 100644 index 000000000000..93375464e5ed --- /dev/null +++ b/components/masking_functions/src/masking_functions/dictionary_flusher_thread.cpp @@ -0,0 +1,388 @@ +/* Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software Foundation, + 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ + +#include "masking_functions/dictionary_flusher_thread.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include // NOLINT(misc-header-include-cycle) + +#include + +#include +#include + +#include +#include +#include + +#include + +#include "masking_functions/abstract_sql_context_builder.hpp" +#include "masking_functions/command_service_tuple.hpp" +#include "masking_functions/primitive_singleton.hpp" +#include "masking_functions/server_helpers.hpp" +#include "masking_functions/term_cache.hpp" +#include "masking_functions/term_cache_core_fwd.hpp" +#ifndef NDEBUG +#include "masking_functions/sql_context.hpp" +#endif +#include "masking_functions/static_sql_context_builder.hpp" + +#include "sql/debug_sync.h" // IWYU pragma: keep + +extern REQUIRES_SERVICE_PLACEHOLDER(mysql_current_thread_reader); + +namespace { + +using global_command_services = masking_functions::primitive_singleton< + masking_functions::command_service_tuple>; + +// an auxilary RAII class that is intended to be used inside 'jthread' handler +// function: it calls 'init()' method from the 'mysql_command_thread' service +// in constructor and 'end()' method from the 'mysql_command_thread' service +// in destructor +class thread_handler_context { + public: + thread_handler_context() { + if ((*global_command_services::instance().thread->init)() != 0) { + throw std::runtime_error{"Cannot initialize thread handler context"}; + } + } + + thread_handler_context(const thread_handler_context &) = delete; + thread_handler_context &operator=(const thread_handler_context &) = delete; + thread_handler_context(thread_handler_context &&) = delete; + thread_handler_context &operator=(thread_handler_context &&) = delete; + + ~thread_handler_context() { + (*global_command_services::instance().thread->end)(); + } +}; + +// an auxiliary RAII class that wraps an instance of 'my_thread_attr_t': it +// calls 'my_thread_attr_init()' in constructor and 'my_thread_attr_destroy()' +// in destructor +class thread_attributes { + public: + thread_attributes() { + if (my_thread_attr_init(&attributes_) != 0) { + throw std::runtime_error{"Cannot initialize thread attributes"}; + } + } + + thread_attributes(const thread_attributes &) = delete; + thread_attributes &operator=(const thread_attributes &) = delete; + thread_attributes(thread_attributes &&) = delete; + thread_attributes &operator=(thread_attributes &&) = delete; + + ~thread_attributes() { my_thread_attr_destroy(&attributes_); } + + void make_joinable() { + if (my_thread_attr_setdetachstate(&attributes_, + MY_THREAD_CREATE_JOINABLE) != 0) { + throw std::runtime_error{ + "Cannot set joinable state for thread attributes"}; + } + } + const my_thread_attr_t &get_underlying() const noexcept { + return attributes_; + } + + private: + my_thread_attr_t attributes_{}; +}; + +// an auxilary RAII class that spawns a joinable thread in the constructor and +// joins it in destructor +class jthread { + public: + using handler_function_type = std::function; + + // passing 'handler_function' deliberately by value to move from + jthread(handler_function_type handler_function, const char *category_name, + const char *name, const char *os_name) + : handler_function_{std::move(handler_function)}, + psi_thread_key_{PSI_NOT_INSTRUMENTED}, + psi_thread_info_{&psi_thread_key_, name, os_name, + PSI_FLAG_SINGLETON, 0, PSI_DOCUMENT_ME}, + handle_{} { + thread_attributes attributes{}; + attributes.make_joinable(); + + mysql_thread_register(category_name, &psi_thread_info_, 1U); + + if (mysql_thread_create(psi_thread_key_, &handle_, + &attributes.get_underlying(), raw_handler, + this) != 0) { + throw std::runtime_error{"Cannot create jthread"}; + } + } + + jthread(const jthread &) = delete; + jthread &operator=(const jthread &) = delete; + jthread(jthread &&) = delete; + jthread &operator=(jthread &&) = delete; + ~jthread() { my_thread_join(&handle_, nullptr); } + + private: + handler_function_type handler_function_; + + PSI_thread_key psi_thread_key_; + PSI_thread_info psi_thread_info_; + + my_thread_handle handle_; + + static void *raw_handler(void *arg) { + try { + thread_handler_context handler_ctx{}; + const auto *self = static_cast(arg); + (self->handler_function_)(); + } catch (const std::exception &e) { + std::string message{"Exception caught in jthread handler - "}; + message += e.what(); + LogComponentErr(ERROR_LEVEL, ER_LOG_PRINTF_MSG, message.c_str()); + } catch (...) { + LogComponentErr(ERROR_LEVEL, ER_LOG_PRINTF_MSG, + "Unexpected exception caught in jthread handler"); + } + return nullptr; + } +}; + +constexpr char flusher_psi_category_name[]{"masking_functions"}; +constexpr char flusher_psi_thread_info_name[]{"masking_functions_dict_flusher"}; +constexpr char flusher_psi_thread_info_os_name[]{"mf_flusher"}; + +} // anonymous namespace + +namespace masking_functions { + +enum class dictionary_flusher_thread::state_type : std::uint8_t { + initial, + initialization_failure, + operational, + stopped +}; + +void dictionary_flusher_thread::jthread_deleter::operator()( + void *ptr) const noexcept { + std::default_delete{}(static_cast(ptr)); +} + +dictionary_flusher_thread::dictionary_flusher_thread( + const term_cache_core_ptr &cache_core, std::uint64_t flush_interval_seconds) + : cache_core_{cache_core}, + flush_interval_seconds_{flush_interval_seconds}, + state_{state_type::initial}, + thread_impl_{new jthread{ + [this]() { do_periodic_reload(); }, flusher_psi_category_name, + flusher_psi_thread_info_name, flusher_psi_thread_info_os_name}} { + // we do not try to populate dict_cache here immediately as this constructor + // is called from the component initialization method and any call to + // mysql_command_query service may mess up with current THD + + // the cache will be loaded during the first call to one of the dictionary + // functions or by the flusher thread +} + +dictionary_flusher_thread::~dictionary_flusher_thread() { + state_.store(state_type::stopped); + // thread_impl_'s destructor will join the thread +} + +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +void dictionary_flusher_thread::do_periodic_reload() { + // waiting for Session Server availability + static constexpr auto sleep_interval{std::chrono::seconds(1)}; + static constexpr std::size_t max_number_of_attempts{30}; + std::size_t number_of_attempts{0U}; + + while (number_of_attempts < max_number_of_attempts && + state_.load() != state_type::stopped && + srv_session_server_is_available() == 0) { + std::this_thread::sleep_for(sleep_interval); + ++number_of_attempts; + } + + if (state_.load() == state_type::stopped) { + LogComponentErr( + WARNING_LEVEL, ER_LOG_PRINTF_MSG, + "Flusher thread terminated while waiting for session server"); + return; + } + if (number_of_attempts >= max_number_of_attempts) { + LogComponentErr( + ERROR_LEVEL, ER_LOG_PRINTF_MSG, + "Session server is unavailable during flusher thread initialization"); + state_.store(state_type::initialization_failure); + return; + } + + // initializing internal connection (along with THD) + term_cache_ptr cache; + + using optional_string_type = std::optional; + optional_string_type failure_message; + try { + // It is important that 'static_sql_context_builder' is created here + // (inside the background thread) because the "static" instance of the + // 'sql_context', which will be created inside 'static_sql_context_builder' + // constructor, will set the 'MYSQL_COMMAND_LOCAL_THD_HANDLE' option to + // 'nullptr' meaning that internal THD object must be created and + // associated with the current thread. + + // Also, there is also a potential deadlock when the component is + // deinitialized immediately after it is installed (when server shuts + // down). + // In this case component's 'deinit()' function will try to stop this + // dictionary flusher thread by calling its desctuctor, which in turns + // will set the state to 'stopped' and will block on joining the thread. + // However, at the same time there is a chance that this thread hasn't + // progressed to the point when it establishes an internal connection + // (creates a shared instance of 'sql_context' inside the + // 'static_sql_context_builder'). This means that the constructor of + // 'sql_context' will try to lock the service registry to establish an + // internal connection while it is already locked by the server's + // component deinitialization logic. + // To avoid this deadlock we simply create an instance of the + // 'sql_context' class (establish an internal connection) under the + // 'LOCK_server_shutting_down' lock and only if the server hasn't entered + // the SHUTDOWN state ('server_shutting_down' is still false). + // The Server sets 'server_shutting_down' to true under the lock at the + // very beginning of the 'clean_up()' function that also performs + // component deinitialization ('component_infrastructure_deinit()'). + abstract_sql_context_builder_ptr sql_ctx_builder{}; + if (!execute_under_lock_if_not_in_shutdown([&sql_ctx_builder]() { + sql_ctx_builder = std::make_shared( + global_command_services::instance()); + })) { + LogComponentErr(WARNING_LEVEL, ER_LOG_PRINTF_MSG, + "Server shutdown requested while attempting to " + "establish flusher thread connection"); + state_.store(state_type::stopped); + return; + } + cache = std::make_unique(cache_core_, sql_ctx_builder); + + // NOLINTNEXTLINE(cppcoreguidelines-avoid-do-while) + DBUG_EXECUTE_IF("enable_masking_functions_flush_thread_sync", { + const sql_context_ptr sql_ctx{sql_ctx_builder->build()}; + std::string wait_action{ + "SET debug_sync = 'masking_functions_before_cache_reload WAIT_FOR " + "masking_functions_before_cache_reload_signal"}; + std::string signal_action{ + "SET debug_sync = 'masking_functions_after_cache_reload SIGNAL " + "masking_functions_after_cache_reload_signal"}; + DBUG_EXECUTE_IF("enable_masking_functions_flush_thread_double_pass", { + wait_action += " EXECUTE 2"; + signal_action += " EXECUTE 2"; + }); + wait_action += '\''; + signal_action += '\''; + + sql_ctx->execute_dml(wait_action); + sql_ctx->execute_dml(signal_action); + }); + } catch (const std::exception &e) { + failure_message.emplace( + "Exception during flusher thread initialization - "); + failure_message->append(e.what()); + } catch (...) { + failure_message.emplace( + "Unknown exception during flusher thread initialization"); + } + + MYSQL_THD extracted_thd{nullptr}; + mysql_service_mysql_current_thread_reader->get(&extracted_thd); + assert(extracted_thd != nullptr); + + const auto is_terminated_lambda{[thd = extracted_thd, this]() { + if ((*is_killed_hook)(thd) != 0) { + state_.store(state_type::stopped); + } + return state_.load() == state_type::stopped; + }}; + + if (is_terminated_lambda()) { + LogComponentErr( + WARNING_LEVEL, ER_LOG_PRINTF_MSG, + "Flusher thread terminated after creating internal connection"); + return; + } + if (failure_message.has_value()) { + LogComponentErr(ERROR_LEVEL, ER_LOG_PRINTF_MSG, failure_message->c_str()); + state_.store(state_type::initialization_failure); + return; + } + state_.store(state_type::operational); + + const auto flush_interval_duration{ + std::chrono::seconds{flush_interval_seconds_}}; + + auto expires_at{std::chrono::steady_clock::now()}; + while (!is_terminated_lambda()) { + if (std::chrono::steady_clock::now() >= expires_at) { + // NOLINTNEXTLINE(cppcoreguidelines-avoid-do-while) + DBUG_EXECUTE_IF("enable_masking_functions_flush_thread_sync", { + DEBUG_SYNC(extracted_thd, "masking_functions_before_cache_reload"); + }); + + failure_message.reset(); + try { + cache->reload_cache(); + } catch (const std::exception &e) { + failure_message.emplace( + "Exception during reloading dictionary cache - "); + failure_message->append(e.what()); + } catch (...) { + failure_message.emplace( + "Unknown exception during reloading dictionary cache"); + } + if (failure_message.has_value()) { + LogComponentErr(ERROR_LEVEL, ER_LOG_PRINTF_MSG, + failure_message->c_str()); + } + + // NOLINTNEXTLINE(cppcoreguidelines-avoid-do-while) + DBUG_EXECUTE_IF("enable_masking_functions_flush_thread_sync", { + DEBUG_SYNC(extracted_thd, "masking_functions_after_cache_reload"); + }); + + expires_at = std::chrono::steady_clock::now() + flush_interval_duration; + } else { + std::this_thread::sleep_for(sleep_interval); + } + } +} + +} // namespace masking_functions diff --git a/components/masking_functions/src/masking_functions/query_builder.cpp b/components/masking_functions/src/masking_functions/query_builder.cpp index 809606d45622..cde3787e93d6 100644 --- a/components/masking_functions/src/masking_functions/query_builder.cpp +++ b/components/masking_functions/src/masking_functions/query_builder.cpp @@ -13,10 +13,12 @@ along with this program; if not, write to the Free Software Foundation, 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ -#include - #include "masking_functions/query_builder.hpp" +#include +#include + +#include "masking_functions/charset_string_fwd.hpp" #include "masking_functions/sql_escape_functions.hpp" namespace masking_functions { @@ -31,25 +33,16 @@ std::string query_builder::insert_ignore_record( return oss.str(); } -std::string query_builder::select_term_for_dictionary_internal( - const charset_string &dictionary_name, - const charset_string *opt_term) const { +std::string query_builder::select_all_from_dictionary() const { std::ostringstream oss; // In our implementation there is no requirement that the `Term` field in // the `mysql.masking_dictionaries` table must be in `utf8mb4`. So, by // adding CONVERT(Term USING utf8mb4) we support other character sets in // the underlying table as well. - oss << "SELECT " - << "CONVERT(" << get_term_field_name() << " USING " - << default_result_character_set << ") FROM " << get_database_name() << '.' - << get_table_name() << " WHERE " << get_dictionary_field_name() << " = '" - << escape_string(dictionary_name) << '\''; - if (opt_term != nullptr) { - oss << " AND " << get_term_field_name() << " = '" - << escape_string(*opt_term) << '\''; - } else { - oss << " ORDER BY RAND() LIMIT 1"; - } + oss << "SELECT " << "CONVERT(" << get_dictionary_field_name() << " USING " + << default_result_character_set << "), " << "CONVERT(" + << get_term_field_name() << " USING " << default_result_character_set + << ") FROM " << get_database_name() << '.' << get_table_name(); return oss.str(); } diff --git a/components/masking_functions/src/masking_functions/random_string_generators.cpp b/components/masking_functions/src/masking_functions/random_string_generators.cpp index bc18fd28851c..e521139db1c9 100644 --- a/components/masking_functions/src/masking_functions/random_string_generators.cpp +++ b/components/masking_functions/src/masking_functions/random_string_generators.cpp @@ -55,22 +55,26 @@ std::default_random_engine &get_thread_local_prng() { } char calculate_luhn_checksum(const std::string &str) noexcept { - std::size_t checksum = 0, n; + std::size_t checksum = 0; + std::size_t digit = 0; std::size_t check_offset = (str.size() + 1U) % 2U; for (std::size_t i = 0; i < str.size(); i++) { // We can convert to int substracting the ASCII for 0 - n = static_cast(str[i] - '0'); + digit = static_cast(str[i] - '0'); if ((i + check_offset) % 2U == 0) { - n *= 2U; - checksum += n > 9U ? (n - 9U) : n; + digit *= 2U; + checksum += digit > 9U ? (digit - 9U) : digit; } else { - checksum += n; + checksum += digit; } } - return checksum % 10 == 0 - ? '0' - : '0' + static_cast(10U - (checksum % 10)); + char result{'0'}; + if (checksum % 10U != 0U) { + result = static_cast(static_cast( + static_cast('0') + 10U - (checksum % 10U))); + } + return result; } } // anonymous namespace @@ -81,35 +85,44 @@ std::string random_character_class_string(character_class char_class, std::size_t length) { if (length == 0U) return {}; - const std::string_view charset_full{ + static constexpr std::string_view charset_full{ "0123456789" "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz" "0123456789"}; + static constexpr std::size_t number_of_digits{10U}; + static constexpr std::size_t number_of_letters{26U}; std::string_view selected_charset; switch (char_class) { case character_class::lower_alpha: - selected_charset = charset_full.substr(10U + 26U, 26U); + selected_charset = charset_full.substr( + number_of_digits + number_of_letters, number_of_letters); break; case character_class::upper_alpha: - selected_charset = charset_full.substr(10U, 26U); + selected_charset = + charset_full.substr(number_of_digits, number_of_letters); break; case character_class::numeric: - selected_charset = charset_full.substr(0U, 10U); + selected_charset = charset_full.substr(0U, number_of_digits); break; case character_class::alpha: - selected_charset = charset_full.substr(10U, 26U + 26U); + selected_charset = charset_full.substr( + number_of_digits, number_of_letters + number_of_letters); break; case character_class::lower_alpha_numeric: - selected_charset = charset_full.substr(10U + 26U, 26U + 10U); + selected_charset = + charset_full.substr(number_of_digits + number_of_letters, + number_of_letters + number_of_digits); break; case character_class::upper_alpha_numeric: - selected_charset = charset_full.substr(0U, 10U + 26U); + selected_charset = + charset_full.substr(0U, number_of_digits + number_of_letters); break; case character_class::alpha_numeric: - selected_charset = charset_full.substr(0U, 10U + 26U + 26U); + selected_charset = charset_full.substr( + 0U, number_of_digits + number_of_letters + number_of_letters); break; default: assert(false); @@ -136,42 +149,63 @@ std::size_t random_number(std::size_t min, std::size_t max) { } std::string random_canada_sin() { + // Three groups of three digits, e.g., 123-456-789 + static constexpr std::size_t first_delimiter_position{3U}; + static constexpr std::size_t second_delimiter_position{7U}; + std::string str; - str.append(random_numeric_string(3)); - str.append(random_numeric_string(3)); - str.append(random_numeric_string(2)); + str += random_numeric_string(3); + str += random_numeric_string(3); + str += random_numeric_string(2); str += calculate_luhn_checksum(str); - str.insert(6, "-"); - str.insert(3, "-"); + str.insert(second_delimiter_position - 1U, "-"); + str.insert(first_delimiter_position, "-"); return str; } // Validate: https://stevemorse.org/ssn/cc.html std::string random_credit_card() { + static constexpr std::size_t american_express{3U}; + static constexpr std::size_t visa{4U}; + static constexpr std::size_t mastercard{5U}; + static constexpr std::size_t discover{6U}; + + static constexpr std::size_t default_number_of_digits{16U}; + static constexpr std::size_t american_express_number_of_digits{15U}; + + static constexpr std::size_t mastercard_sub_min{1U}; + static constexpr std::size_t mastercard_sub_max{5U}; + std::string str; - switch (random_number(3, 6)) { - case 3: + switch (random_number(american_express, discover)) { + case american_express: // American Express: 1st N 3, 2nd N [4,7], len 15 - str.assign("3") - .append(random_number(0, 1) == 0 ? "4" : "7") - .append(random_numeric_string(12)); + str = "3"; + str += random_number(0U, 1U) == 0 ? '4' : '7'; + str += random_numeric_string(american_express_number_of_digits - + str.size() - 1U); break; - case 4: + case visa: // Visa: 1st N 4, len 16 - str.assign("4").append(random_numeric_string(14)); + str = "4"; + str += random_numeric_string(default_number_of_digits - str.size() - 1U); break; - case 5: + case mastercard: // Master Card: 1st N 5, 2nd N [1,5], len 16 - str.assign("5") - .append(std::to_string(random_number(1, 5))) - .append(random_numeric_string(13)); + str = "5"; + str += + std::to_string(random_number(mastercard_sub_min, mastercard_sub_max)); + str += random_numeric_string(default_number_of_digits - str.size() - 1U); break; - case 6: + case discover: // Discover Card: 1st N 6, 2nd N 0, 3rd N 1, 4th N 1, len 16 - str.assign("6011").append(random_numeric_string(11)); + str = "6011"; + str += random_numeric_string(default_number_of_digits - str.size() - 1U); break; + default: + assert(false); } str += calculate_luhn_checksum(str); @@ -182,7 +216,7 @@ std::string random_credit_card() { std::string random_uuid() { static thread_local boost::uuids::random_generator gen; - auto generated = gen(); + boost::uuids::uuid generated = gen(); return boost::uuids::to_string(generated); } @@ -190,32 +224,42 @@ std::string random_ssn() { // AAA-GG-SSSS // Area, Group number, Serial number // Not valid number: Area: 000, 666, 900-999 - return std::to_string(random_number(900, 999)) - .append("-") - .append(random_numeric_string(2)) - .append("-") - .append(random_numeric_string(4)); + static constexpr std::size_t min_invalid{900U}; + static constexpr std::size_t max_invalid{999U}; + + std::string str{std::to_string(random_number(min_invalid, max_invalid))}; + + str += '-'; + str += random_numeric_string(2); + str += '-'; + str += random_numeric_string(4); + return str; } std::string random_iban(std::string_view country, std::size_t length) { // TODO: consider adding IBAN checksum - return std::string(country).append(random_numeric_string(length)); + std::string str{country}; + str += random_numeric_string(length); + return str; } std::string random_uk_nin() { - return std::string("AA").append(random_numeric_string(6)).append("C"); + // A UK National Insurance number (NINo) is made up of two letters, + // six numbers, and a final letter + static constexpr std::size_t number_of_digits{6U}; + std::string str{"AA"}; + str += random_numeric_string(number_of_digits); + str += 'C'; + return str; } std::string random_us_phone() { // 1-555-AAA-BBBB - - return std::string("1") - .append("-") - .append("555") - .append("-") - .append(random_numeric_string(3)) - .append("-") - .append(random_numeric_string(4)); + std::string str{"1-555-"}; + str += random_numeric_string(3); + str += '-'; + str += random_numeric_string(4); + return str; } } // namespace masking_functions diff --git a/components/masking_functions/src/masking_functions/registration_routines.cpp b/components/masking_functions/src/masking_functions/registration_routines.cpp index 6debeee35c4a..bdb8a1416bdf 100644 --- a/components/masking_functions/src/masking_functions/registration_routines.cpp +++ b/components/masking_functions/src/masking_functions/registration_routines.cpp @@ -19,8 +19,16 @@ #include #include -#include +#include +#include +#include #include +#include +#include +#include +#include + +#include #include @@ -31,16 +39,15 @@ #include #include #include +#include #include #include "masking_functions/charset_string.hpp" #include "masking_functions/charset_string_operations.hpp" -#include "masking_functions/command_service_tuple.hpp" #include "masking_functions/primitive_singleton.hpp" -#include "masking_functions/query_builder.hpp" #include "masking_functions/random_string_generators.hpp" -#include "masking_functions/sql_context.hpp" #include "masking_functions/string_service_tuple.hpp" +#include "masking_functions/term_cache.hpp" extern REQUIRES_SERVICE_PLACEHOLDER(udf_registration); extern REQUIRES_SERVICE_PLACEHOLDER(dynamic_privilege_register); @@ -55,10 +62,8 @@ namespace { using global_string_services = masking_functions::primitive_singleton< masking_functions::string_service_tuple>; -using global_command_services = masking_functions::primitive_singleton< - masking_functions::command_service_tuple>; -using global_query_builder = - masking_functions::primitive_singleton; +using global_term_cache = + masking_functions::primitive_singleton; constexpr std::string_view masking_dictionaries_privilege_name = "MASKING_DICTIONARIES_ADMIN"; @@ -66,19 +71,19 @@ constexpr std::string_view masking_dictionaries_privilege_name = // Returns 'true' if current MySQL user has 'MASKING_DICTIONARIES_ADMIN' // dynamic privilege bool have_masking_admin_privilege() { - THD *thd; - if (mysql_service_mysql_current_thread_reader->get(&thd)) { + THD *thd{nullptr}; + if (mysql_service_mysql_current_thread_reader->get(&thd) != 0) { throw std::runtime_error{"Couldn't query current thd"}; } - Security_context_handle sctx; - if (mysql_service_mysql_thd_security_context->get(thd, &sctx)) { + Security_context_handle sctx{nullptr}; + if (mysql_service_mysql_thd_security_context->get(thd, &sctx) != 0) { throw std::runtime_error{"Couldn't query security context"}; } if (mysql_service_global_grants_check->has_global_grant( sctx, masking_dictionaries_privilege_name.data(), - masking_dictionaries_privilege_name.size())) + masking_dictionaries_privilege_name.size()) != 0) return true; return false; @@ -144,7 +149,7 @@ void set_return_value_collation_from_arg( // If is less then , NULL is returned. class gen_range_impl { public: - gen_range_impl(mysqlpp::udf_context &ctx) { + explicit gen_range_impl(mysqlpp::udf_context &ctx) { if (ctx.get_number_of_args() != 2) throw std::invalid_argument{"Wrong argument list: should be (int, int)"}; @@ -158,15 +163,20 @@ class gen_range_impl { ctx.set_arg_type(1, INT_RESULT); } - mysqlpp::udf_result_t calculate(const mysqlpp::udf_context &ctx) { + // original UDF Wrappers design assumed that calculate() won't be a static + // member function, however making it static does not do any harm as + // static member functions can still be called on an instance of a class + static mysqlpp::udf_result_t calculate( + const mysqlpp::udf_context &ctx) { + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) const auto lower = *ctx.get_arg(0); + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) const auto upper = *ctx.get_arg(1); if (upper < lower) { return std::nullopt; - } else { - return masking_functions::random_number(lower, upper); } + return masking_functions::random_number(lower, upper); } }; @@ -189,7 +199,7 @@ class gen_rnd_email_impl { static constexpr std::size_t max_surname_length = 1024; public: - gen_rnd_email_impl(mysqlpp::udf_context &ctx) { + explicit gen_rnd_email_impl(mysqlpp::udf_context &ctx) { if (ctx.get_number_of_args() > 3) throw std::invalid_argument{ "Wrong argument list: should be ([int], [int], [string])"}; @@ -222,7 +232,7 @@ class gen_rnd_email_impl { } } - mysqlpp::udf_result_t calculate( + static mysqlpp::udf_result_t calculate( const mysqlpp::udf_context &ctx) { masking_functions::charset_string cs_email_domain; if (ctx.get_number_of_args() >= 3) { @@ -233,9 +243,12 @@ class gen_rnd_email_impl { masking_functions::charset_string::default_collation_name}; } - const long long name_length = ctx.get_number_of_args() >= 1 - ? *ctx.get_arg(0) - : default_name_length; + // NOLINTBEGIN(bugprone-unchecked-optional-access) + const long long name_length = + ctx.get_number_of_args() >= 1 + ? *ctx.get_arg(0) + : static_cast(default_name_length); + // NOLINTEND(bugprone-unchecked-optional-access) if (name_length <= 0) { throw std::invalid_argument{"Name length must be a positive number"}; } @@ -245,9 +258,12 @@ class gen_rnd_email_impl { std::to_string(max_name_length)}; } - const long long surname_length = ctx.get_number_of_args() >= 2 - ? *ctx.get_arg(1) - : default_surname_length; + // NOLINTBEGIN(bugprone-unchecked-optional-access) + const long long surname_length = + ctx.get_number_of_args() >= 2 + ? *ctx.get_arg(1) + : static_cast(default_surname_length); + // NOLINTEND(bugprone-unchecked-optional-access) if (surname_length <= 0) { throw std::invalid_argument{"Surname length must be a positive number"}; } @@ -312,7 +328,7 @@ class gen_rnd_iban_impl { } public: - gen_rnd_iban_impl(mysqlpp::udf_context &ctx) { + explicit gen_rnd_iban_impl(mysqlpp::udf_context &ctx) { if (ctx.get_number_of_args() > 2) throw std::invalid_argument{ "Wrong argument list: should be ([string], [int])"}; @@ -340,7 +356,7 @@ class gen_rnd_iban_impl { } } - mysqlpp::udf_result_t calculate( + static mysqlpp::udf_result_t calculate( const mysqlpp::udf_context &ctx) { masking_functions::charset_string cs_country_code; if (ctx.get_number_of_args() >= 1) { @@ -359,9 +375,12 @@ class gen_rnd_iban_impl { conversion_buffer); validate_ansi_country_code(ascii_country_code); - const long long iban_length = ctx.get_number_of_args() >= 2 - ? *ctx.get_arg(1) - : default_number_of_characters; + // NOLINTBEGIN(bugprone-unchecked-optional-access) + const long long iban_length = + ctx.get_number_of_args() >= 2 + ? *ctx.get_arg(1) + : static_cast(default_number_of_characters); + // NOLINTEND(bugprone-unchecked-optional-access) if (iban_length < 0) { throw std::invalid_argument{"IBAN length must not be a negative number"}; @@ -392,7 +411,7 @@ class gen_rnd_iban_impl { class rnd_impl_base { public: - rnd_impl_base(mysqlpp::udf_context &ctx) { + explicit rnd_impl_base(mysqlpp::udf_context &ctx) { if (ctx.get_number_of_args() != 0) { throw std::invalid_argument{"Wrong argument list: should be empty"}; } @@ -401,9 +420,6 @@ class rnd_impl_base { charset_ext.set_return_value_collation( ctx, masking_functions::charset_string::default_collation_name); } - - protected: - ~rnd_impl_base() = default; }; // @@ -416,7 +432,8 @@ class rnd_impl_base { class gen_rnd_canada_sin_impl final : private rnd_impl_base { public: using rnd_impl_base::rnd_impl_base; - mysqlpp::udf_result_t calculate(const mysqlpp::udf_context &) { + static mysqlpp::udf_result_t calculate( + const mysqlpp::udf_context & /*unused*/) { return masking_functions::random_canada_sin(); } }; @@ -430,7 +447,8 @@ class gen_rnd_canada_sin_impl final : private rnd_impl_base { class gen_rnd_pan_impl final : private rnd_impl_base { public: using rnd_impl_base::rnd_impl_base; - mysqlpp::udf_result_t calculate(const mysqlpp::udf_context &) { + static mysqlpp::udf_result_t calculate( + const mysqlpp::udf_context & /*unused*/) { return masking_functions::random_credit_card(); } }; @@ -444,7 +462,8 @@ class gen_rnd_pan_impl final : private rnd_impl_base { class gen_rnd_ssn_impl final : private rnd_impl_base { public: using rnd_impl_base::rnd_impl_base; - mysqlpp::udf_result_t calculate(const mysqlpp::udf_context &) { + static mysqlpp::udf_result_t calculate( + const mysqlpp::udf_context & /*unused*/) { return masking_functions::random_ssn(); } }; @@ -458,7 +477,8 @@ class gen_rnd_ssn_impl final : private rnd_impl_base { class gen_rnd_uk_nin_impl final : private rnd_impl_base { public: using rnd_impl_base::rnd_impl_base; - mysqlpp::udf_result_t calculate(const mysqlpp::udf_context &) { + static mysqlpp::udf_result_t calculate( + const mysqlpp::udf_context & /*unused*/) { return masking_functions::random_uk_nin(); } }; @@ -471,7 +491,8 @@ class gen_rnd_uk_nin_impl final : private rnd_impl_base { class gen_rnd_us_phone_impl final : private rnd_impl_base { public: using rnd_impl_base::rnd_impl_base; - mysqlpp::udf_result_t calculate(const mysqlpp::udf_context &) { + static mysqlpp::udf_result_t calculate( + const mysqlpp::udf_context & /*unused*/) { return masking_functions::random_us_phone(); } }; @@ -484,7 +505,8 @@ class gen_rnd_us_phone_impl final : private rnd_impl_base { class gen_rnd_uuid_impl final : private rnd_impl_base { public: using rnd_impl_base::rnd_impl_base; - mysqlpp::udf_result_t calculate(const mysqlpp::udf_context &) { + static mysqlpp::udf_result_t calculate( + const mysqlpp::udf_context & /*unused*/) { return masking_functions::random_uuid(); } }; @@ -498,7 +520,7 @@ class gen_rnd_uuid_impl final : private rnd_impl_base { // Optional is used as a masking character. class mask_inner_impl { public: - mask_inner_impl(mysqlpp::udf_context &ctx) { + explicit mask_inner_impl(mysqlpp::udf_context &ctx) { if (ctx.get_number_of_args() < 3 || ctx.get_number_of_args() > 4) throw std::invalid_argument{ "Wrong argument list: should be (string, int, int, [char])"}; @@ -525,7 +547,7 @@ class mask_inner_impl { set_return_value_collation_from_arg(charset_ext, ctx, 0); } - mysqlpp::udf_result_t calculate( + static mysqlpp::udf_result_t calculate( const mysqlpp::udf_context &ctx) { if (ctx.is_arg_null(0)) return std::nullopt; @@ -534,7 +556,9 @@ class mask_inner_impl { const auto masking_char = determine_masking_char(ctx, 3, x_ascii_masking_char); + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) const auto left_margin = *ctx.get_arg(1); + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) const auto right_margin = *ctx.get_arg(2); if (left_margin < 0 || right_margin < 0) { @@ -559,7 +583,7 @@ class mask_inner_impl { // Optional is used as a masking character. class mask_outer_impl { public: - mask_outer_impl(mysqlpp::udf_context &ctx) { + explicit mask_outer_impl(mysqlpp::udf_context &ctx) { if (ctx.get_number_of_args() < 3 || ctx.get_number_of_args() > 4) throw std::invalid_argument{ "Wrong argument list: should be (string, int, int [char])"}; @@ -586,7 +610,7 @@ class mask_outer_impl { set_return_value_collation_from_arg(charset_ext, ctx, 0); } - mysqlpp::udf_result_t calculate( + static mysqlpp::udf_result_t calculate( const mysqlpp::udf_context &ctx) { if (ctx.is_arg_null(0)) return std::nullopt; @@ -595,7 +619,9 @@ class mask_outer_impl { const auto masking_char = determine_masking_char(ctx, 3, x_ascii_masking_char); + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) const auto left_margin = *ctx.get_arg(1); + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) const auto right_margin = *ctx.get_arg(2); if (left_margin < 0 || right_margin < 0) { @@ -633,7 +659,7 @@ class mask_impl_base { const masking_functions::charset_string &masking_char) const = 0; public: - mask_impl_base(mysqlpp::udf_context &ctx) { + explicit mask_impl_base(mysqlpp::udf_context &ctx) { if (ctx.get_number_of_args() < 1 || ctx.get_number_of_args() > 2) throw std::invalid_argument{ "Wrong argument list: should be (string, [char])"}; @@ -666,11 +692,10 @@ class mask_impl_base { throw std::invalid_argument{"Argument must be exactly " + std::to_string(min_length()) + " characters"}; - } else { - throw std::invalid_argument{ - "Argument must be between " + std::to_string(min_length()) + - " and " + std::to_string(max_length()) + " characters"}; } + throw std::invalid_argument{"Argument must be between " + + std::to_string(min_length()) + " and " + + std::to_string(max_length()) + " characters"}; } const auto masking_char = @@ -682,6 +707,11 @@ class mask_impl_base { } protected: + mask_impl_base(const mask_impl_base &) = default; + mask_impl_base &operator=(const mask_impl_base &) = default; + mask_impl_base(mask_impl_base &&) = default; + mask_impl_base &operator=(mask_impl_base &&) = default; + ~mask_impl_base() = default; }; @@ -701,13 +731,16 @@ class mask_canada_sin_impl final : private mask_impl_base { using mask_impl_base::mask_impl_base; private: - virtual std::size_t min_length() const override { return 9; } - virtual std::size_t max_length() const override { return 11; } - virtual std::string_view default_ascii_masking_char() const override { + static constexpr std::size_t min_length_value{9U}; + static constexpr std::size_t max_length_value{11U}; + + std::size_t min_length() const override { return min_length_value; } + std::size_t max_length() const override { return max_length_value; } + std::string_view default_ascii_masking_char() const override { return x_ascii_masking_char; } - virtual masking_functions::charset_string process( + masking_functions::charset_string process( const masking_functions::charset_string &cs_str, const masking_functions::charset_string &masking_char) const override { if (cs_str.get_size_in_characters() == max_length()) { @@ -718,11 +751,10 @@ class mask_canada_sin_impl final : private mask_impl_base { auto sresult = masking_functions::mask_inner(cs_str, 4, 4, masking_char); sresult = masking_functions::mask_inner(sresult, 0, 8, masking_char); return masking_functions::mask_inner(sresult, 8, 0, masking_char); - } else { - // otherwise (no delimiters at all, or just one at an unknown position), - // we use 'mask_inner_alphanum()' for the whole range - return masking_functions::mask_inner_alphanum(cs_str, 0, 0, masking_char); } + // otherwise (no delimiters at all, or just one at an unknown position), + // we use 'mask_inner_alphanum()' for the whole range + return masking_functions::mask_inner_alphanum(cs_str, 0, 0, masking_char); } }; @@ -743,12 +775,15 @@ class mask_iban_impl final : private mask_impl_base { using mask_impl_base::mask_impl_base; private: - virtual std::size_t min_length() const override { return 15; } - virtual std::size_t max_length() const override { return 34 + 8; } - virtual std::string_view default_ascii_masking_char() const override { + static constexpr std::size_t min_length_value{15U}; + static constexpr std::size_t max_length_value{34U + 8U}; + + std::size_t min_length() const override { return min_length_value; } + std::size_t max_length() const override { return max_length_value; } + std::string_view default_ascii_masking_char() const override { return star_ascii_masking_char; } - virtual masking_functions::charset_string process( + masking_functions::charset_string process( const masking_functions::charset_string &cs_str, const masking_functions::charset_string &masking_char) const override { // as positions of the delimiters may vary, we always use the @@ -774,13 +809,16 @@ class mask_pan_impl final : private mask_impl_base { using mask_impl_base::mask_impl_base; private: - virtual std::size_t min_length() const override { return 14; } - virtual std::size_t max_length() const override { return 19; } - virtual std::string_view default_ascii_masking_char() const override { + static constexpr std::size_t min_length_value{14U}; + static constexpr std::size_t max_length_value{19U}; + + std::size_t min_length() const override { return min_length_value; } + std::size_t max_length() const override { return max_length_value; } + std::string_view default_ascii_masking_char() const override { return x_ascii_masking_char; } - virtual masking_functions::charset_string process( + masking_functions::charset_string process( const masking_functions::charset_string &cs_str, const masking_functions::charset_string &masking_char) const override { // as positions of the delimiters may vary, we always use the @@ -806,18 +844,24 @@ class mask_pan_relaxed_impl final : private mask_impl_base { using mask_impl_base::mask_impl_base; private: - virtual std::size_t min_length() const override { return 14; } - virtual std::size_t max_length() const override { return 19; } - virtual std::string_view default_ascii_masking_char() const override { + static constexpr std::size_t min_length_value{14U}; + static constexpr std::size_t max_length_value{19U}; + + std::size_t min_length() const override { return min_length_value; } + std::size_t max_length() const override { return max_length_value; } + std::string_view default_ascii_masking_char() const override { return x_ascii_masking_char; } - virtual masking_functions::charset_string process( + masking_functions::charset_string process( const masking_functions::charset_string &cs_str, const masking_functions::charset_string &masking_char) const override { // as positions of the delimiters may vary, we always use the // 'mask_inner_alphanum()' function for everything except the first 6 and // the last 4 characters - return masking_functions::mask_inner_alphanum(cs_str, 6, 4, masking_char); + static constexpr std::size_t prefix_length{6U}; + static constexpr std::size_t suffix_length{4U}; + return masking_functions::mask_inner_alphanum(cs_str, prefix_length, + suffix_length, masking_char); } }; @@ -837,12 +881,15 @@ class mask_ssn_impl final : private mask_impl_base { using mask_impl_base::mask_impl_base; private: - virtual std::size_t min_length() const override { return 9; } - virtual std::size_t max_length() const override { return 11; } - virtual std::string_view default_ascii_masking_char() const override { + static constexpr std::size_t min_length_value{9U}; + static constexpr std::size_t max_length_value{11U}; + + std::size_t min_length() const override { return min_length_value; } + std::size_t max_length() const override { return max_length_value; } + std::string_view default_ascii_masking_char() const override { return star_ascii_masking_char; } - virtual masking_functions::charset_string process( + masking_functions::charset_string process( const masking_functions::charset_string &cs_str, const masking_functions::charset_string &masking_char) const override { if (cs_str.get_size_in_characters() == max_length()) { @@ -850,13 +897,25 @@ class mask_ssn_impl final : private mask_impl_base { // delimiters to be at the predefined positions (after the first 3 // digits and before the last 4 digits), we just mask everything except // the delimiters - auto sresult = masking_functions::mask_inner(cs_str, 4, 5, masking_char); - return masking_functions::mask_inner(sresult, 0, 8, masking_char); - } else { - // otherwise (no delimiters at all, or just one at an unknown position), - // we use 'mask_inner_alphanum()' for everything except the last 4 digits - return masking_functions::mask_inner_alphanum(cs_str, 0, 4, masking_char); + static constexpr std::size_t long_first_prefix_length{4U}; + static constexpr std::size_t long_first_suffix_length{5U}; + auto sresult = + masking_functions::mask_inner(cs_str, long_first_prefix_length, + long_first_suffix_length, masking_char); + + static constexpr std::size_t long_second_prefix_length{0U}; + static constexpr std::size_t long_second_suffix_length{8U}; + return masking_functions::mask_inner(sresult, long_second_prefix_length, + long_second_suffix_length, + masking_char); } + // otherwise (no delimiters at all, or just one at an unknown position), + // we use 'mask_inner_alphanum()' for everything except the last 4 digits + static constexpr std::size_t short_second_prefix_length{0U}; + static constexpr std::size_t short_second_suffix_length{4U}; + return masking_functions::mask_inner_alphanum( + cs_str, short_second_prefix_length, short_second_suffix_length, + masking_char); } }; @@ -876,12 +935,15 @@ class mask_uk_nin_impl final : private mask_impl_base { using mask_impl_base::mask_impl_base; private: - virtual std::size_t min_length() const override { return 9; } - virtual std::size_t max_length() const override { return 11; } - virtual std::string_view default_ascii_masking_char() const override { + static constexpr std::size_t min_length_value{9U}; + static constexpr std::size_t max_length_value{11U}; + + std::size_t min_length() const override { return min_length_value; } + std::size_t max_length() const override { return max_length_value; } + std::string_view default_ascii_masking_char() const override { return star_ascii_masking_char; } - virtual masking_functions::charset_string process( + masking_functions::charset_string process( const masking_functions::charset_string &cs_str, const masking_functions::charset_string &masking_char) const override { // as positions of the delimiters may vary, we always use the @@ -907,12 +969,15 @@ class mask_uuid_impl final : private mask_impl_base { using mask_impl_base::mask_impl_base; private: - virtual std::size_t min_length() const override { return 36; } - virtual std::size_t max_length() const override { return 36; } - virtual std::string_view default_ascii_masking_char() const override { + static constexpr std::size_t min_length_value{36U}; + static constexpr std::size_t max_length_value{36U}; + + std::size_t min_length() const override { return min_length_value; } + std::size_t max_length() const override { return max_length_value; } + std::string_view default_ascii_masking_char() const override { return star_ascii_masking_char; } - virtual masking_functions::charset_string process( + masking_functions::charset_string process( const masking_functions::charset_string &cs_str, const masking_functions::charset_string &masking_char) const override { return masking_functions::mask_inner_alphanum(cs_str, 0, 0, masking_char); @@ -930,7 +995,7 @@ class mask_uuid_impl final : private mask_impl_base { // via the third argument, otherwise the original term is returned. class gen_blocklist_impl { public: - gen_blocklist_impl(mysqlpp::udf_context &ctx) { + explicit gen_blocklist_impl(mysqlpp::udf_context &ctx) { if (ctx.get_number_of_args() != 3) throw std::invalid_argument{ "Wrong argument list: gen_blocklist(string, string, string)"}; @@ -955,7 +1020,7 @@ class gen_blocklist_impl { set_return_value_collation_from_arg(charset_ext, ctx, 0); } - mysqlpp::udf_result_t calculate( + static mysqlpp::udf_result_t calculate( const mysqlpp::udf_context &ctx) { if (ctx.is_arg_null(0)) return std::nullopt; @@ -963,38 +1028,24 @@ class gen_blocklist_impl { const auto cs_dict_a = make_charset_string_from_arg(ctx, 1); const auto cs_dict_b = make_charset_string_from_arg(ctx, 2); - { - masking_functions::sql_context sql_ctx{ - global_command_services::instance()}; - - auto query = - global_query_builder::instance().check_term_presence_in_dictionary( - cs_dict_a, cs_term); - auto sresult = sql_ctx.query_single_value(query); - - if (!sresult) { - return {std::string{cs_term.get_buffer()}}; - } + if (!global_term_cache::instance()->contains(cs_dict_a, cs_term)) { + return {std::string{cs_term.get_buffer()}}; } - masking_functions::sql_context sql_ctx{global_command_services::instance()}; - - auto query = - global_query_builder::instance().select_random_term_for_dictionary( - cs_dict_b); - auto sresult = sql_ctx.query_single_value(query); - - if (sresult && sresult->size() > 0) { - masking_functions::charset_string utf8_result{ - global_string_services::instance(), *sresult, - masking_functions::charset_string::utf8mb4_collation_name}; - masking_functions::charset_string conversion_buffer; - const auto &cs_result = masking_functions::smart_convert_to_collation( - utf8_result, cs_term.get_collation(), conversion_buffer); - return {std::string{cs_result.get_buffer()}}; - } else { + const auto cs_random_term = + global_term_cache::instance()->get_random(cs_dict_b); + + if (cs_random_term.is_default_constructed()) { return std::nullopt; } + // random term should be in utf8mb4 + assert(cs_random_term.get_collation() == + masking_functions::charset_string::get_utf8mb4_collation( + global_string_services::instance())); + masking_functions::charset_string conversion_buffer; + const auto &cs_result = masking_functions::smart_convert_to_collation( + cs_random_term, cs_term.get_collation(), conversion_buffer); + return {std::string{cs_result.get_buffer()}}; } }; @@ -1005,7 +1056,7 @@ class gen_blocklist_impl { // specified via the first argument. class gen_dictionary_impl { public: - gen_dictionary_impl(mysqlpp::udf_context &ctx) { + explicit gen_dictionary_impl(mysqlpp::udf_context &ctx) { if (ctx.get_number_of_args() != 1) throw std::invalid_argument{ "Wrong argument list: gen_dictionary(string)"}; @@ -1023,22 +1074,53 @@ class gen_dictionary_impl { ctx, masking_functions::charset_string::default_collation_name); } - mysqlpp::udf_result_t calculate( + static mysqlpp::udf_result_t calculate( const mysqlpp::udf_context &ctx) { const auto cs_dictionary = make_charset_string_from_arg(ctx, 0); + const auto cs_random_term = + global_term_cache::instance()->get_random(cs_dictionary); - masking_functions::sql_context sql_ctx{global_command_services::instance()}; + if (cs_random_term.is_default_constructed()) { + return std::nullopt; + } - auto query = - global_query_builder::instance().select_random_term_for_dictionary( - cs_dictionary); - auto sresult = sql_ctx.query_single_value(query); + // random term should already be in utf8mb4 + assert(cs_random_term.get_collation() == + masking_functions::charset_string::get_utf8mb4_collation( + global_string_services::instance())); + return {std::string{cs_random_term.get_buffer()}}; + } +}; - if (sresult && sresult->size() > 0) { - return *sresult; - } else { - return std::nullopt; +// +// masking_dictionaries_flush() +// +// Flush the data from the masking dictionaries table to the memory cache. +class masking_dictionaries_flush_impl { + public: + explicit masking_dictionaries_flush_impl(mysqlpp::udf_context &ctx) { + if (!have_masking_admin_privilege()) { + throw std::invalid_argument{ + "Function requires " + + std::string(masking_dictionaries_privilege_name) + " privilege"}; } + + if (ctx.get_number_of_args() > 0) + throw std::invalid_argument{ + "Wrong argument list: masking_dictionaries_flush()"}; + + ctx.mark_result_nullable(false); + // Calling this UDF two or more times has exactly the same effect as just + // calling it once. So, we mark the result as 'const' here so that the + // optimizer could use this info to eliminate unnecessary calls. + ctx.mark_result_const(true); + } + + static mysqlpp::udf_result_t calculate( + const mysqlpp::udf_context &ctx [[maybe_unused]]) { + global_term_cache::instance()->reload_cache(); + + return 1; } }; @@ -1049,7 +1131,7 @@ class gen_dictionary_impl { // argument, and all of its terms from the dictionary registry. class masking_dictionary_remove_impl { public: - masking_dictionary_remove_impl(mysqlpp::udf_context &ctx) { + explicit masking_dictionary_remove_impl(mysqlpp::udf_context &ctx) { if (!have_masking_admin_privilege()) { throw std::invalid_argument{ "Function requires " + @@ -1060,7 +1142,7 @@ class masking_dictionary_remove_impl { throw std::invalid_argument{ "Wrong argument list: masking_dictionary_remove(string)"}; - ctx.mark_result_nullable(true); + ctx.mark_result_nullable(false); // Calling this UDF two or more times has exactly the same effect as just // calling it once. So, we mark the result as 'const' here so that the // optimizer could use this info to eliminate unnecessary calls. @@ -1069,26 +1151,13 @@ class masking_dictionary_remove_impl { // arg0 - dictionary ctx.mark_arg_nullable(0, false); ctx.set_arg_type(0, STRING_RESULT); - - mysqlpp::udf_context_charset_extension charset_ext{ - mysql_service_mysql_udf_metadata}; - charset_ext.set_return_value_collation( - ctx, masking_functions::charset_string::default_collation_name); } - mysqlpp::udf_result_t calculate( + static mysqlpp::udf_result_t calculate( const mysqlpp::udf_context &ctx) { const auto cs_dictionary = make_charset_string_from_arg(ctx, 0); - masking_functions::sql_context sql_ctx{global_command_services::instance()}; - - auto query = - global_query_builder::instance().delete_for_dictionary(cs_dictionary); - if (!sql_ctx.execute(query)) { - return std::nullopt; - } else { - return "1"; - } + return global_term_cache::instance()->remove(cs_dictionary) ? 1 : 0; } }; @@ -1099,7 +1168,7 @@ class masking_dictionary_remove_impl { // dictionary whose name is specified via the first argument. class masking_dictionary_term_add_impl { public: - masking_dictionary_term_add_impl(mysqlpp::udf_context &ctx) { + explicit masking_dictionary_term_add_impl(mysqlpp::udf_context &ctx) { if (!have_masking_admin_privilege()) { throw std::invalid_argument{ "Function requires " + @@ -1111,7 +1180,7 @@ class masking_dictionary_term_add_impl { "Wrong argument list: masking_dictionary_term_add(string, " "string)"}; - ctx.mark_result_nullable(true); + ctx.mark_result_nullable(false); // Calling this UDF two or more times has exactly the same effect as just // calling it once. So, we mark the result as 'const' here so that the // optimizer could use this info to eliminate unnecessary calls. @@ -1124,28 +1193,15 @@ class masking_dictionary_term_add_impl { // arg1 - term ctx.mark_arg_nullable(1, false); ctx.set_arg_type(1, STRING_RESULT); - - mysqlpp::udf_context_charset_extension charset_ext{ - mysql_service_mysql_udf_metadata}; - charset_ext.set_return_value_collation( - ctx, masking_functions::charset_string::default_collation_name); } - mysqlpp::udf_result_t calculate( + static mysqlpp::udf_result_t calculate( const mysqlpp::udf_context &ctx) { const auto cs_dictionary = make_charset_string_from_arg(ctx, 0); const auto cs_term = make_charset_string_from_arg(ctx, 1); - masking_functions::sql_context sql_ctx{global_command_services::instance()}; - - auto query = global_query_builder::instance().insert_ignore_record( - cs_dictionary, cs_term); - - if (!sql_ctx.execute(query)) { - return std::nullopt; - } else { - return "1"; - } + return global_term_cache::instance()->insert(cs_dictionary, cs_term) ? 1 + : 0; } }; @@ -1156,7 +1212,7 @@ class masking_dictionary_term_add_impl { // dictionary whose name is specified via the first argument. class masking_dictionary_term_remove_impl { public: - masking_dictionary_term_remove_impl(mysqlpp::udf_context &ctx) { + explicit masking_dictionary_term_remove_impl(mysqlpp::udf_context &ctx) { if (!have_masking_admin_privilege()) { throw std::invalid_argument{ "Function requires " + @@ -1168,7 +1224,7 @@ class masking_dictionary_term_remove_impl { "Wrong argument list: masking_dictionary_term_remove(string, " "string)"}; - ctx.mark_result_nullable(true); + ctx.mark_result_nullable(false); // Calling this UDF two or more times has exactly the same effect as just // calling it once. So, we mark the result as 'const' here so that the // optimizer could use this info to eliminate unnecessary calls. @@ -1181,28 +1237,15 @@ class masking_dictionary_term_remove_impl { // arg1 - term ctx.mark_arg_nullable(1, false); ctx.set_arg_type(1, STRING_RESULT); - - mysqlpp::udf_context_charset_extension charset_ext{ - mysql_service_mysql_udf_metadata}; - charset_ext.set_return_value_collation( - ctx, masking_functions::charset_string::default_collation_name); } - mysqlpp::udf_result_t calculate( + static mysqlpp::udf_result_t calculate( const mysqlpp::udf_context &ctx) { const auto cs_dictionary = make_charset_string_from_arg(ctx, 0); const auto cs_term = make_charset_string_from_arg(ctx, 1); - masking_functions::sql_context sql_ctx{global_command_services::instance()}; - - auto query = - global_query_builder::instance().delete_for_dictionary_and_term( - cs_dictionary, cs_term); - if (!sql_ctx.execute(query)) { - return std::nullopt; - } else { - return "1"; - } + return global_term_cache::instance()->remove(cs_dictionary, cs_term) ? 1 + : 0; } }; @@ -1229,12 +1272,14 @@ DECLARE_STRING_UDF_AUTO(mask_uk_nin) DECLARE_STRING_UDF_AUTO(mask_uuid) DECLARE_STRING_UDF_AUTO(gen_blocklist) DECLARE_STRING_UDF_AUTO(gen_dictionary) -DECLARE_STRING_UDF_AUTO(masking_dictionary_remove) -DECLARE_STRING_UDF_AUTO(masking_dictionary_term_add) -DECLARE_STRING_UDF_AUTO(masking_dictionary_term_remove) +DECLARE_INT_UDF_AUTO(masking_dictionaries_flush) +DECLARE_INT_UDF_AUTO(masking_dictionary_remove) +DECLARE_INT_UDF_AUTO(masking_dictionary_term_add) +DECLARE_INT_UDF_AUTO(masking_dictionary_term_remove) // TODO: in c++20 (where CTAD works for alias templates) this shoud be changed // to 'static const udf_info_container known_udfs' +// NOLINTBEGIN(cppcoreguidelines-pro-type-cstyle-cast) std::array known_udfs{DECLARE_UDF_INFO_AUTO(gen_range), DECLARE_UDF_INFO_AUTO(gen_rnd_email), DECLARE_UDF_INFO_AUTO(gen_rnd_iban), @@ -1256,15 +1301,21 @@ std::array known_udfs{DECLARE_UDF_INFO_AUTO(gen_range), DECLARE_UDF_INFO_AUTO(mask_uuid), DECLARE_UDF_INFO_AUTO(gen_blocklist), DECLARE_UDF_INFO_AUTO(gen_dictionary), + DECLARE_UDF_INFO_AUTO(masking_dictionaries_flush), DECLARE_UDF_INFO_AUTO(masking_dictionary_remove), DECLARE_UDF_INFO_AUTO(masking_dictionary_term_add), DECLARE_UDF_INFO_AUTO(masking_dictionary_term_remove)}; +// NOLINTEND(cppcoreguidelines-pro-type-cstyle-cast) + +namespace { using udf_bitset_type = mysqlpp::udf_bitset>; -static udf_bitset_type registered_udfs; +udf_bitset_type registered_udfs; + +bool privileges_registered = false; -static bool privileges_registered = false; +} // anonymous namespace namespace masking_functions { diff --git a/components/masking_functions/src/masking_functions/server_helpers.cpp b/components/masking_functions/src/masking_functions/server_helpers.cpp new file mode 100644 index 000000000000..f5797104c1a3 --- /dev/null +++ b/components/masking_functions/src/masking_functions/server_helpers.cpp @@ -0,0 +1,37 @@ +/* Copyright (c) 2023 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software Foundation, + 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ + +#include "masking_functions/server_helpers.hpp" + +#include + +#include "sql/mysqld.h" + +namespace masking_functions { + +// this function is put into a separate translation unit as it uses internal +// 'mysqld' interface - 'sql/mysqld.h' +bool execute_under_lock_if_not_in_shutdown( + const lock_protected_function &func) { + rwlock_scoped_lock rdlock(&LOCK_server_shutting_down, false, __FILE__, + __LINE__); + if (server_shutting_down) { + return false; + } + func(); + return true; +} + +} // namespace masking_functions diff --git a/components/masking_functions/src/masking_functions/sql_context.cpp b/components/masking_functions/src/masking_functions/sql_context.cpp index b351c0568f0c..b6bc5f85d0ed 100644 --- a/components/masking_functions/src/masking_functions/sql_context.cpp +++ b/components/masking_functions/src/masking_functions/sql_context.cpp @@ -13,21 +13,27 @@ along with this program; if not, write to the Free Software Foundation, 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ +#include #include +#include +#include #include -#include #include +#include +#include #include +#include -#include "masking_functions/sql_context.hpp" +#include // needed only for MYSQL_ERRMSG_SIZE + +#include #include "masking_functions/command_service_tuple.hpp" +#include "masking_functions/sql_context.hpp" namespace { -MYSQL_H to_mysql_h(void *p) noexcept { return static_cast(p); } - -constexpr const char default_command_user_name[] = "root"; +MYSQL_H to_mysql_h(void *ptr) noexcept { return static_cast(ptr); } } // anonymous namespace @@ -37,39 +43,61 @@ void sql_context::deleter::operator()(void *ptr) const noexcept { if (ptr != nullptr) (*services->factory->close)(to_mysql_h(ptr)); } -sql_context::sql_context(const command_service_tuple &services) +sql_context::sql_context( + const command_service_tuple &services, + sql_context_registry_access initialization_registry_locking_mode, + sql_context_registry_access operation_registry_locking_mode, + bool initialize_thread) : impl_{nullptr, deleter{&services}} { MYSQL_H local_mysql_h = nullptr; if ((*get_services().factory->init)(&local_mysql_h) != 0) { - throw std::runtime_error{"Couldn't initialize server handle"}; + raise_with_error_message("Couldn't initialize server handle"); } assert(local_mysql_h != nullptr); impl_.reset(local_mysql_h); + if (initialize_thread) { + if ((*get_services().options->set)( + local_mysql_h, MYSQL_COMMAND_LOCAL_THD_HANDLE, nullptr) != 0) { + raise_with_error_message("Couldn't set local THD handle"); + } + } + + // setting MYSQL_NO_LOCK_REGISTRY is needed for cases when we destroy + // 'sql_context' from the 'UNINSTALL COMPONENT' handler (component's + // 'deinit()' function) + const bool initialization_no_lock_registry_option_value{ + initialization_registry_locking_mode == + sql_context_registry_access::non_locking}; + if ((*get_services().options->set)( + local_mysql_h, MYSQL_NO_LOCK_REGISTRY, + &initialization_no_lock_registry_option_value) != 0) { + raise_with_error_message( + "Couldn't set initialization registry locking mode"); + } + // setting MYSQL_COMMAND_PROTOCOL to nullptr will be translated to the // default value "local" if ((*get_services().options->set)(local_mysql_h, MYSQL_COMMAND_PROTOCOL, nullptr) != 0) { - throw std::runtime_error{"Couldn't set protocol"}; + raise_with_error_message("Couldn't set protocol"); } - // setting MYSQL_COMMAND_USER_NAME to default_command_user_name here - // as the default MYSQL_SESSION_USER ("mysql.session") does not have - // access to the mysql.masking_dictionaries + // nullptr here will be translated into MYSQL_SESSION_USER ("mysql.session") if ((*get_services().options->set)(local_mysql_h, MYSQL_COMMAND_USER_NAME, - default_command_user_name) != 0) { - throw std::runtime_error{"Couldn't set username"}; + nullptr) != 0) { + raise_with_error_message("Couldn't set username"); } // setting MYSQL_COMMAND_HOST_NAME to nullptr will be translated to the // default MYSQL_SYS_HOST ("localhost") if ((*get_services().options->set)(local_mysql_h, MYSQL_COMMAND_HOST_NAME, nullptr) != 0) { - throw std::runtime_error{"Couldn't set hostname"}; + raise_with_error_message("Couldn't set hostname"); } if ((*get_services().factory->connect)(local_mysql_h) != 0) { - throw std::runtime_error{"Couldn't establish server connection"}; + raise_with_error_message("Couldn't establish server connection"); } // In order to make sure that internal INSERT / DELETE queries which @@ -77,25 +105,70 @@ sql_context::sql_context(const command_service_tuple &services) // value of '@@global.autocommit' (we want all operations to be committed // immediately), we are setting the value of the 'autocommit' session // variable here explicitly to 'ON'. - if ((*get_services().factory->autocommit)(to_mysql_h(impl_.get()), true)) { - throw std::runtime_error{"Couldn't set autocommit"}; + if ((*get_services().factory->autocommit)(local_mysql_h, true) != 0) { + raise_with_error_message("Couldn't set autocommit"); + } + + if (operation_registry_locking_mode != initialization_registry_locking_mode) { + const bool operation_no_lock_registry_option_value{ + operation_registry_locking_mode == + sql_context_registry_access::non_locking}; + if ((*get_services().options->set)( + local_mysql_h, MYSQL_NO_LOCK_REGISTRY, + &operation_no_lock_registry_option_value) != 0) { + raise_with_error_message("Couldn't set operation registry locking mode"); + } + } +} + +void sql_context::reset() { + if ((*get_services().factory->reset)(to_mysql_h(impl_.get())) != 0) { + raise_with_error_message("Couldn't reset connection"); + } +} + +bool sql_context::execute_dml(std::string_view query) { + // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto) + const auto casted_impl{to_mysql_h(impl_.get())}; + if ((*get_services().query->query)(casted_impl, query.data(), + query.length()) != 0) { + raise_with_error_message("Error while executing SQL DML query"); + } + std::uint64_t row_count = 0; + if ((*get_services().query->affected_rows)(casted_impl, &row_count) != 0) { + raise_with_error_message("Couldn't get number of affected rows"); } + return row_count > 0; } -sql_context::optional_string sql_context::query_single_value( - std::string_view query) { - if ((*get_services().query->query)(to_mysql_h(impl_.get()), query.data(), +void sql_context::execute_select_internal( + std::string_view query, std::size_t expected_number_of_fields, + const row_internal_callback &callback) { + // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto) + const auto casted_impl{to_mysql_h(impl_.get())}; + if ((*get_services().query->query)(casted_impl, query.data(), query.length()) != 0) { - throw std::runtime_error{"Error while executing SQL query"}; + raise_with_error_message("Error while executing SQL select query"); + } + + unsigned int actual_number_of_fields = 0; + if ((*get_services().field_info->field_count)( + casted_impl, &actual_number_of_fields) != 0) { + raise_with_error_message("Couldn't get number of fields"); + } + + if (actual_number_of_fields != expected_number_of_fields) { + raise_with_error_message( + "Mismatch between actual and expected number of fields"); } MYSQL_RES_H mysql_res = nullptr; - if ((*get_services().query_result->store_result)(to_mysql_h(impl_.get()), - &mysql_res) != 0) { - throw std::runtime_error{"Couldn't store MySQL result"}; + if ((*get_services().query_result->store_result)(casted_impl, &mysql_res) != + 0) { + raise_with_error_message("Couldn't store MySQL result"); } if (mysql_res == nullptr) { - throw std::runtime_error{"Couldn't create MySQL result handler"}; + raise_with_error_message("Couldn't create MySQL result handler"); } auto mysql_res_deleter = [deleter = get_services().query_result->free_result]( @@ -107,41 +180,62 @@ sql_context::optional_string sql_context::query_single_value( std::unique_ptr; mysql_res_ptr mysql_res_guard(mysql_res, std::move(mysql_res_deleter)); - uint64_t row_count = 0; + std::uint64_t row_count = 0; // As the 'affected_rows()' method of the 'mysql_command_query' MySQL - // service is implementted via 'mysql_affected_rows()' MySQL client + // service is implemented via 'mysql_affected_rows()' MySQL client // function, it is OK to use it for SELECT statements as well, because // in this case it will work like 'mysql_num_rows()'. - if ((*get_services().query->affected_rows)(to_mysql_h(impl_.get()), - &row_count) != 0) - throw std::runtime_error{"Couldn't query row count"}; - - if (row_count == 0) return std::nullopt; + if ((*get_services().query->affected_rows)(casted_impl, &row_count) != 0) + raise_with_error_message("Couldn't query row count"); - if (row_count > 1) throw std::runtime_error{"Query returned more than 1 row"}; + for (std::uint64_t i = 0; i < row_count; ++i) { + MYSQL_ROW_H field_values = nullptr; + ulong *field_value_lengths = nullptr; // NOLINT(misc-include-cleaner) - MYSQL_ROW_H row = nullptr; - if ((*get_services().query_result->fetch_row)(mysql_res, &row) != 0) - throw std::runtime_error{"Couldn't fetch row"}; + if ((*get_services().query_result->fetch_row)(mysql_res, &field_values) != + 0) + raise_with_error_message("Couldn't fetch row"); + if ((*get_services().query_result->fetch_lengths)( + mysql_res, &field_value_lengths) != 0) + raise_with_error_message("Couldn't fetch length"); - ulong *length = nullptr; - if ((*get_services().query_result->fetch_lengths)(mysql_res, &length) != 0) - throw std::runtime_error{"Couldn't fetch lenghts"}; - - return optional_string{std::in_place, row[0], length[0]}; + callback(field_values, field_value_lengths); + } } -bool sql_context::execute(std::string_view query) { - if ((*get_services().query->query)(to_mysql_h(impl_.get()), query.data(), - query.length()) != 0) { - return false; - } - uint64_t row_count = 0; - if ((*get_services().query->affected_rows)(to_mysql_h(impl_.get()), - &row_count) != 0) { - return false; +[[noreturn]] void sql_context::raise_with_error_message( + std::string_view prefix) { + std::string message{prefix}; + + // despite the fact that sql_error service method expects 'char **ptr' as an + // output parameter, it does not do '*ptr = some_internal_string', instead it + // expects this double pointer to point to a pointer to a valid buffer and + // does 'memcpy(*ptr, ...)' + + // unfortunately, there is no way to specify the size of the buffer - + // however, as this buffer is filled with the strings coming from the + // 'mysql_error()' client API function, its max size is known to be + // MYSQL_ERRMSG_SIZE + unsigned int error_number{0}; + using error_message_buffer_type = std::array; + error_message_buffer_type error_message_buffer; + char *error_message_buffer_ptr{std::data(error_message_buffer)}; + // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto) + const auto casted_impl{to_mysql_h(impl_.get())}; + if (casted_impl != nullptr && + (*get_services().error_info->sql_errno)(casted_impl, &error_number) == + 0 && + (*get_services().error_info->sql_error)(casted_impl, + &error_message_buffer_ptr) == 0) { + message += "(errno = "; + message += std::to_string(error_number); + message += " \""; + if (error_message_buffer_ptr != nullptr) { + message += error_message_buffer_ptr; + } + message += "\")"; } - return row_count > 0; + throw std::runtime_error{message}; } } // namespace masking_functions diff --git a/components/masking_functions/src/masking_functions/sql_escape_functions.cpp b/components/masking_functions/src/masking_functions/sql_escape_functions.cpp index a71f246d73e2..9d33f96d74df 100644 --- a/components/masking_functions/src/masking_functions/sql_escape_functions.cpp +++ b/components/masking_functions/src/masking_functions/sql_escape_functions.cpp @@ -18,6 +18,7 @@ #include #include +#include #include #include "masking_functions/charset_string.hpp" @@ -40,12 +41,12 @@ std::string escape_string(const charset_string &cs_str) { const auto *cs = get_charset_by_name(charset_string::utf8mb4_collation_name, MYF(0)); assert(cs != nullptr); - std::size_t r = escape_string_for_mysql(cs, res.data(), max_size, - buffer.data(), buffer.size()); - if (r == ~static_cast(0)) + std::size_t escaping_result = escape_string_for_mysql( + cs, res.data(), max_size, buffer.data(), buffer.size()); + if (escaping_result == ~static_cast(0)) throw std::runtime_error{"cannot escape string for sql"}; - res.resize(r); + res.resize(escaping_result); return res; } diff --git a/components/masking_functions/src/masking_functions/static_sql_context_builder.cpp b/components/masking_functions/src/masking_functions/static_sql_context_builder.cpp new file mode 100644 index 000000000000..929a5a519f55 --- /dev/null +++ b/components/masking_functions/src/masking_functions/static_sql_context_builder.cpp @@ -0,0 +1,37 @@ +/* Copyright (c) 2023 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software Foundation, + 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ + +#include "masking_functions/static_sql_context_builder.hpp" + +#include + +#include "masking_functions/command_service_tuple_fwd.hpp" +#include "masking_functions/sql_context.hpp" + +namespace masking_functions { + +static_sql_context_builder::static_sql_context_builder( + const command_service_tuple &services) + : abstract_sql_context_builder{services}, + static_instance_{std::make_shared( + get_services(), sql_context_registry_access::locking, + sql_context_registry_access::non_locking, true)} {} +static_sql_context_builder::~static_sql_context_builder() = default; + +sql_context_ptr static_sql_context_builder::do_build() const { + return static_instance_; +} + +} // namespace masking_functions diff --git a/components/masking_functions/src/masking_functions/sys_vars.cpp b/components/masking_functions/src/masking_functions/sys_vars.cpp new file mode 100644 index 000000000000..41ceac079bc7 --- /dev/null +++ b/components/masking_functions/src/masking_functions/sys_vars.cpp @@ -0,0 +1,129 @@ +/* Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software Foundation, + 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ + +#include "masking_functions/sys_vars.hpp" + +#include +#include +#include +#include +#include + +#include + +// #include +#include +#include + +#include "masking_functions/component_sys_variable_service_tuple.hpp" +#include "masking_functions/primitive_singleton.hpp" + +namespace { + +using global_component_sys_variable_services = + masking_functions::primitive_singleton< + masking_functions::component_sys_variable_service_tuple>; + +constexpr const char component_name[]{"masking_functions"}; +constexpr const char masking_database_var_name[]{"masking_database"}; +constexpr const char flush_interval_var_name[]{ + "dictionaries_flush_interval_seconds"}; + +char default_database_name[]{"mysql"}; +const ulonglong default_flush_interval_seconds = 0; + +bool is_database_name_initialised = false; +bool is_flush_interval_initialised = false; + +char *database_name; +ulonglong flush_interval_seconds = 0; + +} // anonymous namespace + +namespace masking_functions { + +std::string_view get_dict_database_name() noexcept { return database_name; } + +std::uint64_t get_flush_interval_seconds() noexcept { + return flush_interval_seconds; +} + +bool register_sys_vars() { + STR_CHECK_ARG(str) check_db_name{default_database_name}; + + const auto &services{global_component_sys_variable_services::instance()}; + // NOLINTNEXTLINE(bugprone-multi-level-implicit-pointer-conversion) + void *casted_database_name{static_cast(&database_name)}; + if (services.registrator->register_variable( + component_name, masking_database_var_name, + PLUGIN_VAR_STR | PLUGIN_VAR_MEMALLOC | PLUGIN_VAR_RQCMDARG | + PLUGIN_VAR_READONLY, + "Specifies the database to use for data masking dictionaries " + "at server startup.", + nullptr, nullptr, static_cast(&check_db_name), + casted_database_name) != 0) { + return false; + } + is_database_name_initialised = true; + + INTEGRAL_CHECK_ARG(ulonglong) + check_flush_interval{default_flush_interval_seconds, 0, ULLONG_MAX, 1}; + + if (services.registrator->register_variable( + component_name, flush_interval_var_name, + PLUGIN_VAR_LONGLONG | PLUGIN_VAR_UNSIGNED | PLUGIN_VAR_RQCMDARG | + PLUGIN_VAR_READONLY, + "Sets the interval, in seconds, to wait before attempting to " + "schedule another flush of the data masking dictionaries table to " + "the memory data masking dictionaries cache following a restart or " + "previous execution.", + nullptr, nullptr, static_cast(&check_flush_interval), + static_cast(&flush_interval_seconds)) != 0) { + return false; + } + is_flush_interval_initialised = true; + + return true; +} + +bool unregister_sys_vars() { + bool is_success = true; + + const auto &services{global_component_sys_variable_services::instance()}; + if (is_database_name_initialised && + services.unregistrator->unregister_variable( + component_name, masking_database_var_name) != 0) { + is_success = false; + } + + if (is_flush_interval_initialised && + services.unregistrator->unregister_variable( + component_name, flush_interval_var_name) != 0) { + is_success = false; + } + + return is_success; +} + +bool check_sys_vars(std::string &error_message) { + if (database_name == nullptr || std::strlen(database_name) == 0) { + error_message = "Bad masking_functions.masking_database value"; + return false; + } + + return true; +} + +} // namespace masking_functions diff --git a/components/masking_functions/src/masking_functions/term_cache.cpp b/components/masking_functions/src/masking_functions/term_cache.cpp new file mode 100644 index 000000000000..d47fb0db552c --- /dev/null +++ b/components/masking_functions/src/masking_functions/term_cache.cpp @@ -0,0 +1,56 @@ +/* Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software Foundation, + 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ + +#include "masking_functions/term_cache.hpp" + +#include "masking_functions/abstract_sql_context_builder_fwd.hpp" +#include "masking_functions/charset_string.hpp" +#include "masking_functions/term_cache_core.hpp" + +namespace masking_functions { + +term_cache::term_cache(const term_cache_core_ptr &core, + const abstract_sql_context_builder_ptr &sql_ctx_builder) + : core_{core}, sql_ctx_builder_{sql_ctx_builder} {} + +term_cache::~term_cache() = default; + +bool term_cache::contains(const charset_string &dictionary_name, + const charset_string &term) const { + return core_->contains(*sql_ctx_builder_, dictionary_name, term); +} + +charset_string term_cache::get_random( + const charset_string &dictionary_name) const { + return core_->get_random(*sql_ctx_builder_, dictionary_name); +} + +bool term_cache::remove(const charset_string &dictionary_name) { + return core_->remove(*sql_ctx_builder_, dictionary_name); +} + +bool term_cache::remove(const charset_string &dictionary_name, + const charset_string &term) { + return core_->remove(*sql_ctx_builder_, dictionary_name, term); +} + +bool term_cache::insert(const charset_string &dictionary_name, + const charset_string &term) { + return core_->insert(*sql_ctx_builder_, dictionary_name, term); +} + +void term_cache::reload_cache() { core_->reload_cache(*sql_ctx_builder_); } + +} // namespace masking_functions diff --git a/components/masking_functions/src/masking_functions/term_cache_core.cpp b/components/masking_functions/src/masking_functions/term_cache_core.cpp new file mode 100644 index 000000000000..34642c10833f --- /dev/null +++ b/components/masking_functions/src/masking_functions/term_cache_core.cpp @@ -0,0 +1,237 @@ +/* Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software Foundation, + 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ + +#include "masking_functions/term_cache_core.hpp" + +#include +#include +#include +#include +#include + +#include "masking_functions/abstract_sql_context_builder.hpp" +#include "masking_functions/bookshelf.hpp" +#include "masking_functions/charset_string.hpp" +#include "masking_functions/primitive_singleton.hpp" +#include "masking_functions/query_builder.hpp" +#include "masking_functions/sql_context.hpp" +#include "masking_functions/string_service_tuple.hpp" + +namespace masking_functions { + +using global_query_builder = masking_functions::primitive_singleton< + masking_functions::query_builder_ptr>; + +term_cache_core::term_cache_core() = default; + +term_cache_core::~term_cache_core() = default; + +bool term_cache_core::contains( + const abstract_sql_context_builder &sql_ctx_builder, + const charset_string &dictionary_name, const charset_string &term) const { + sql_context_ptr sql_ctx; + shared_lock_type read_lock{}; + unique_lock_type write_lock{}; + const auto &acquired_dict_cache{acquire_dict_cache_shared( + sql_ctx_builder, sql_ctx, read_lock, write_lock)}; + + charset_string dictionary_name_buffer; + const auto &utf8mb4_dictionary_name{ + to_utf8mb4(dictionary_name, dictionary_name_buffer)}; + charset_string term_buffer; + const auto &utf8mb4_term{to_utf8mb4(term, term_buffer)}; + return acquired_dict_cache.contains(utf8mb4_dictionary_name, utf8mb4_term); +} + +charset_string term_cache_core::get_random( + const abstract_sql_context_builder &sql_ctx_builder, + const charset_string &dictionary_name) const { + sql_context_ptr sql_ctx; + shared_lock_type read_lock{}; + unique_lock_type write_lock{}; + const auto &acquired_dict_cache{acquire_dict_cache_shared( + sql_ctx_builder, sql_ctx, read_lock, write_lock)}; + + charset_string dictionary_name_buffer; + const auto &utf8mb4_dictionary_name{ + to_utf8mb4(dictionary_name, dictionary_name_buffer)}; + return acquired_dict_cache.get_random(utf8mb4_dictionary_name); +} + +bool term_cache_core::remove( + const abstract_sql_context_builder &sql_ctx_builder, + const charset_string &dictionary_name) { + sql_context_ptr sql_ctx; + auto query{ + global_query_builder::instance()->delete_for_dictionary(dictionary_name)}; + + unique_lock_type write_lock{}; + auto &acquired_dict_cache{ + acquire_dict_cache_unique(sql_ctx_builder, sql_ctx, write_lock)}; + + if (!sql_ctx) { + sql_ctx = sql_ctx_builder.build(); + } + + // there is a chance that a user can delete the dictionary from the + // dictionary table directly (not via UDF function) and execute_dml() + // will return false here, whereas cache operation will return true - + // this is why we rely only on the result of the cache operation + sql_ctx->execute_dml(query); + charset_string dictionary_name_buffer; + const auto &utf8mb4_dictionary_name{ + to_utf8mb4(dictionary_name, dictionary_name_buffer)}; + return acquired_dict_cache.remove(utf8mb4_dictionary_name); +} + +bool term_cache_core::remove( + const abstract_sql_context_builder &sql_ctx_builder, + const charset_string &dictionary_name, const charset_string &term) { + sql_context_ptr sql_ctx; + auto query{global_query_builder::instance()->delete_for_dictionary_and_term( + dictionary_name, term)}; + + unique_lock_type write_lock{}; + auto &acquired_dict_cache{ + acquire_dict_cache_unique(sql_ctx_builder, sql_ctx, write_lock)}; + + if (!sql_ctx) { + sql_ctx = sql_ctx_builder.build(); + } + + // similarly to another remove() method, we ignore the result of the + // sql operation and rely only on the result of the cache modification + sql_ctx->execute_dml(query); + charset_string dictionary_name_buffer; + const auto &utf8mb4_dictionary_name{ + to_utf8mb4(dictionary_name, dictionary_name_buffer)}; + charset_string term_buffer; + const auto &utf8mb4_term{to_utf8mb4(term, term_buffer)}; + return acquired_dict_cache.remove(utf8mb4_dictionary_name, utf8mb4_term); +} + +bool term_cache_core::insert( + const abstract_sql_context_builder &sql_ctx_builder, + const charset_string &dictionary_name, const charset_string &term) { + sql_context_ptr sql_ctx; + auto query{global_query_builder::instance()->insert_ignore_record( + dictionary_name, term)}; + + unique_lock_type write_lock{}; + auto &acquired_dict_cache{ + acquire_dict_cache_unique(sql_ctx_builder, sql_ctx, write_lock)}; + + if (!sql_ctx) { + sql_ctx = sql_ctx_builder.build(); + } + + // here, as cache insert may throw, we start the 2-phase operation + // with this cache insert because it can be easily reversed without throwing + charset_string dictionary_name_buffer; + const auto &utf8mb4_dictionary_name{ + to_utf8mb4(dictionary_name, dictionary_name_buffer)}; + charset_string term_buffer; + const auto &utf8mb4_term{to_utf8mb4(term, term_buffer)}; + const auto result{ + acquired_dict_cache.insert(utf8mb4_dictionary_name, utf8mb4_term)}; + try { + sql_ctx->execute_dml(query); + } catch (...) { + dict_cache_->remove(utf8mb4_dictionary_name, utf8mb4_term); + throw; + } + + return result; +} + +void term_cache_core::reload_cache( + const abstract_sql_context_builder &sql_ctx_builder) { + unique_lock_type dict_cache_write_lock{dict_cache_mutex_}; + + std::string error_message; + auto sql_ctx{sql_ctx_builder.build()}; + auto local_dict_cache{create_dict_cache_internal(*sql_ctx, error_message)}; + if (!local_dict_cache) { + throw std::runtime_error{error_message}; + } + + dict_cache_ = std::move(local_dict_cache); +} + +bookshelf_ptr term_cache_core::create_dict_cache_internal( + sql_context &sql_ctx, std::string &error_message) { + bookshelf_ptr result; + error_message.clear(); + try { + auto query{global_query_builder::instance()->select_all_from_dictionary()}; + auto local_dict_cache{std::make_unique()}; + sql_context::row_callback<2> result_inserter{[&terms = *local_dict_cache]( + const auto &field_values) { + const auto &string_services{ + primitive_singleton::instance()}; + charset_string dictionary_name{string_services, field_values[0], + charset_string::utf8mb4_collation_name}; + charset_string term{string_services, field_values[1], + charset_string::utf8mb4_collation_name}; + terms.insert(dictionary_name, term); + }}; + sql_ctx.execute_select(query, result_inserter); + result = std::move(local_dict_cache); + } catch (const std::exception &e) { + error_message = e.what(); + } catch (...) { + error_message = + "unexpected exception caught while loading dictionary cache"; + } + + return result; +} + +const bookshelf &term_cache_core::acquire_dict_cache_shared( + const abstract_sql_context_builder &sql_ctx_builder, + sql_context_ptr &sql_ctx, shared_lock_type &read_lock, + unique_lock_type &write_lock) const { + read_lock = shared_lock_type{dict_cache_mutex_}; + if (!dict_cache_) { + // upgrading to a unique_lock + read_lock.unlock(); + acquire_dict_cache_unique(sql_ctx_builder, sql_ctx, write_lock); + } + return *dict_cache_; +} + +bookshelf &term_cache_core::acquire_dict_cache_unique( + const abstract_sql_context_builder &sql_ctx_builder, + sql_context_ptr &sql_ctx, unique_lock_type &write_lock) const { + write_lock = unique_lock_type{dict_cache_mutex_}; + if (!dict_cache_) { + std::string error_message; + sql_ctx = sql_ctx_builder.build(); + auto local_dict_cache{create_dict_cache_internal(*sql_ctx, error_message)}; + if (!local_dict_cache) { + throw std::runtime_error{error_message}; + } + dict_cache_ = std::move(local_dict_cache); + } + return *dict_cache_; +} + +const charset_string &term_cache_core::to_utf8mb4(const charset_string &str, + charset_string &buffer) { + return smart_convert_to_collation(str, charset_string::utf8mb4_collation_name, + buffer); +} + +} // namespace masking_functions diff --git a/mysql-test/include/have_masking_functions_component.inc b/mysql-test/include/have_masking_functions_component.inc index d54039e2fdaa..3db23e784947 100644 --- a/mysql-test/include/have_masking_functions_component.inc +++ b/mysql-test/include/have_masking_functions_component.inc @@ -8,7 +8,7 @@ if (!$MASKING_FUNCTIONS_COMPONENT) { } # -## Check if --plugin-dir was setup for component_encryption_udf +## Check if --plugin-dir was setup for component_masking_functions # if (`SELECT CONCAT('--plugin-dir=', REPLACE(@@plugin_dir, '\\\\', '/')) != '$MASKING_FUNCTIONS_COMPONENT_OPT/'`) { --skip component_masking_functions requires that --plugin-dir is set to the component_masking_functions dir (the .opt file does not contain \$MASKING_FUNCTIONS_COMPONENT_OPT) diff --git a/mysql-test/suite/component_masking_functions/r/dictionary_ai_ci_lookups.result b/mysql-test/suite/component_masking_functions/r/dictionary_ai_ci_lookups.result new file mode 100644 index 000000000000..4057c785eb0e --- /dev/null +++ b/mysql-test/suite/component_masking_functions/r/dictionary_ai_ci_lookups.result @@ -0,0 +1,43 @@ +# +# Checking for proper case-insensitive 'utf8mb4_0900_ai_ci' string +# comparison in Masking Function dictionary operations +INSTALL COMPONENT 'file://component_masking_functions'; +CREATE TABLE mysql.masking_dictionaries( +Dictionary VARCHAR(256) NOT NULL, +Term VARCHAR(256) NOT NULL, +UNIQUE INDEX dictionary_term_idx (Dictionary, Term) +) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4; +GRANT SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries TO 'mysql.session'@'localhost'; +CREATE USER udftest_priv@localhost; +GRANT MASKING_DICTIONARIES_ADMIN ON *.* TO udftest_priv@localhost; +# +# Polulating dictionary 'dict' +include/assert.inc [The very first masking_dictionary_term_add() must succeed] +# +# Checking case-sensitivity and accents in dictionary terms +include/assert.inc [Terms with differences in case must be considered equal] +include/assert.inc [Terms with differences in accent must be considered equal] +include/assert.inc [Terms with differences in both case and accent must be considered equal] +include/assert.inc [Terms with German Eszett and "ss" must be considered equal] +include/assert.inc [Terms with German Capital Eszett and "ss" must be considered equal] +# +# Checking case-sensitivity and accents in dictionary names +include/assert.inc [Dictionary names with differences in case must be considered equal] +include/assert.inc [Dictionary names with differences in accent must be considered equal] +include/assert.inc [Dictionary names with differences in both case and accent must be considered equal] +# +# Polulating dictionary 'dummy' (will be used for substitutions) +include/assert.inc [The second meaningful call to masking_dictionary_term_add() must succeed] +# +# Checking Dictionary Cache case-insensitive accent-aware lookups via gen_blocklist() +include/assert.inc [Original term must be substituted] +include/assert.inc [Term with changed case must be substituted] +include/assert.inc [Term with changed accent must be substituted] +include/assert.inc [Term with both changed case and changed accent must be substituted] +include/assert.inc [Terms with German Eszett must be substituted] +include/assert.inc [Terms with German Capital Eszett must be substituted] +include/assert.inc [Terms with actual differences must not be substituted] +DROP USER udftest_priv@localhost; +UNINSTALL COMPONENT 'file://component_masking_functions'; +REVOKE SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries FROM 'mysql.session'@'localhost'; +DROP TABLE mysql.masking_dictionaries; diff --git a/mysql-test/suite/component_masking_functions/r/dictionary_operations.result b/mysql-test/suite/component_masking_functions/r/dictionary_operations.result index 5b2ea684ffbf..57d7812363a8 100644 --- a/mysql-test/suite/component_masking_functions/r/dictionary_operations.result +++ b/mysql-test/suite/component_masking_functions/r/dictionary_operations.result @@ -5,9 +5,16 @@ # * masking_dictionary_term_add # * masking_dictionary_term_remove # * masking_dictionary_remove +# * masking_dictionaries_flush # INSTALL COMPONENT 'file://component_masking_functions'; -# arity checks +# +# Create users with and without MASKING_DICTIONARIES_ADMIN privilege +CREATE USER udftest_unpriv@localhost; +CREATE USER udftest_priv@localhost; +GRANT MASKING_DICTIONARIES_ADMIN ON *.* TO udftest_priv@localhost; +# +# arity checks, run for privileged user SELECT gen_dictionary(); ERROR HY000: Can't initialize function 'gen_dictionary'; Wrong argument list: gen_dictionary(string) SELECT gen_dictionary('', ''); @@ -21,21 +28,24 @@ ERROR HY000: Can't initialize function 'gen_blocklist'; Wrong argument list: gen SELECT gen_blocklist('', '', '', ''); ERROR HY000: Can't initialize function 'gen_blocklist'; Wrong argument list: gen_blocklist(string, string, string) SELECT masking_dictionary_term_add(); -ERROR HY000: Can't initialize function 'masking_dictionary_term_add'; Function requires MASKING_DICTIONARIES_ADMIN privilege +ERROR HY000: Can't initialize function 'masking_dictionary_term_add'; Wrong argument list: masking_dictionary_term_add(string, string) SELECT masking_dictionary_term_add(''); -ERROR HY000: Can't initialize function 'masking_dictionary_term_add'; Function requires MASKING_DICTIONARIES_ADMIN privilege +ERROR HY000: Can't initialize function 'masking_dictionary_term_add'; Wrong argument list: masking_dictionary_term_add(string, string) SELECT masking_dictionary_term_add('', '', ''); -ERROR HY000: Can't initialize function 'masking_dictionary_term_add'; Function requires MASKING_DICTIONARIES_ADMIN privilege +ERROR HY000: Can't initialize function 'masking_dictionary_term_add'; Wrong argument list: masking_dictionary_term_add(string, string) SELECT masking_dictionary_term_remove(); -ERROR HY000: Can't initialize function 'masking_dictionary_term_remove'; Function requires MASKING_DICTIONARIES_ADMIN privilege +ERROR HY000: Can't initialize function 'masking_dictionary_term_remove'; Wrong argument list: masking_dictionary_term_remove(string, string) SELECT masking_dictionary_term_remove(''); -ERROR HY000: Can't initialize function 'masking_dictionary_term_remove'; Function requires MASKING_DICTIONARIES_ADMIN privilege +ERROR HY000: Can't initialize function 'masking_dictionary_term_remove'; Wrong argument list: masking_dictionary_term_remove(string, string) SELECT masking_dictionary_term_remove('', '', ''); -ERROR HY000: Can't initialize function 'masking_dictionary_term_remove'; Function requires MASKING_DICTIONARIES_ADMIN privilege +ERROR HY000: Can't initialize function 'masking_dictionary_term_remove'; Wrong argument list: masking_dictionary_term_remove(string, string) SELECT masking_dictionary_remove(); -ERROR HY000: Can't initialize function 'masking_dictionary_remove'; Function requires MASKING_DICTIONARIES_ADMIN privilege +ERROR HY000: Can't initialize function 'masking_dictionary_remove'; Wrong argument list: masking_dictionary_remove(string) SELECT masking_dictionary_remove('', ''); -ERROR HY000: Can't initialize function 'masking_dictionary_remove'; Function requires MASKING_DICTIONARIES_ADMIN privilege +ERROR HY000: Can't initialize function 'masking_dictionary_remove'; Wrong argument list: masking_dictionary_remove(string) +SELECT masking_dictionaries_flush(''); +ERROR HY000: Can't initialize function 'masking_dictionaries_flush'; Wrong argument list: masking_dictionaries_flush() +# # argument nullness checks for functions not requiring MASKING_DICTIONARIES_ADMIN SELECT gen_dictionary(NULL); ERROR HY000: gen_dictionary UDF failed; argument 1 cannot be null @@ -43,37 +53,57 @@ SELECT gen_blocklist('Berlin', NULL, 'us_cities'); ERROR HY000: gen_blocklist UDF failed; argument 2 cannot be null SELECT gen_blocklist('Berlin', 'de_cities', NULL); ERROR HY000: gen_blocklist UDF failed; argument 3 cannot be null +# # checking the case when there is no mysql.masking_dictionaries table -SELECT gen_blocklist('Berlin', 'de_cities', 'us_cities'); -ERROR HY000: Error in command service backend interface, because of : "Table 'mysql.masking_dictionaries' doesn't exist" +GRANT CREATE, SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries TO 'mysql.session'@'localhost'; SELECT gen_dictionary('us_cities'); ERROR HY000: Error in command service backend interface, because of : "Table 'mysql.masking_dictionaries' doesn't exist" +SELECT masking_dictionaries_flush(); +ERROR HY000: Error in command service backend interface, because of : "Table 'mysql.masking_dictionaries' doesn't exist" +SELECT masking_dictionary_term_add('single_dict', 'entry'); +ERROR HY000: Error in command service backend interface, because of : "Table 'mysql.masking_dictionaries' doesn't exist" +REVOKE CREATE, SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries FROM 'mysql.session'@'localhost'; +# # NULL for NULL checks include/assert.inc [gen_blocklist() for the NULL primary argument should return NULL] +# # checking the case when mysql.masking_dictionaries has invalid structure CREATE TABLE mysql.masking_dictionaries( Dictionary VARCHAR(256) NOT NULL, UNIQUE INDEX dictionary_term_idx (Dictionary) ) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4; +GRANT SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries TO 'mysql.session'@'localhost'; SELECT gen_blocklist('Berlin', 'de_cities', 'us_cities'); ERROR HY000: Error in command service backend interface, because of : "Unknown column 'Term' in 'field list'" SELECT gen_dictionary('us_cities'); ERROR HY000: Error in command service backend interface, because of : "Unknown column 'Term' in 'field list'" +REVOKE SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries FROM 'mysql.session'@'localhost'; DROP TABLE mysql.masking_dictionaries; +# # checks for an unprivileged user -CREATE USER udftest; SELECT masking_dictionary_term_add('single_dict', 'entry'); ERROR HY000: Can't initialize function 'masking_dictionary_term_add'; Function requires MASKING_DICTIONARIES_ADMIN privilege SELECT masking_dictionary_term_remove('single_dict', 'entry'); ERROR HY000: Can't initialize function 'masking_dictionary_term_remove'; Function requires MASKING_DICTIONARIES_ADMIN privilege SELECT masking_dictionary_remove('single_dict'); ERROR HY000: Can't initialize function 'masking_dictionary_remove'; Function requires MASKING_DICTIONARIES_ADMIN privilege -# checking the case when mysql.masking_dictionaries is empty +# +# checking the case when 'mysql.session'@'localhost' has insufficient privileges CREATE TABLE mysql.masking_dictionaries( Dictionary VARCHAR(256) NOT NULL, Term VARCHAR(256) NOT NULL, UNIQUE INDEX dictionary_term_idx (Dictionary, Term) ) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4; +SELECT gen_dictionary('us_cities'); +ERROR HY000: Error in command service backend interface, because of : "SELECT command denied to user 'mysql.session'@'localhost' for table 'masking_dictionaries'" +SELECT masking_dictionaries_flush(); +ERROR HY000: Error in command service backend interface, because of : "SELECT command denied to user 'mysql.session'@'localhost' for table 'masking_dictionaries'" +GRANT SELECT ON mysql.masking_dictionaries TO 'mysql.session'@'localhost'; +SELECT masking_dictionary_term_add('single_dict', 'entry'); +ERROR HY000: Error in command service backend interface, because of : "INSERT command denied to user 'mysql.session'@'localhost' for table 'masking_dictionaries'" +# +# checking the case when mysql.masking_dictionaries is empty +GRANT INSERT, UPDATE, DELETE ON mysql.masking_dictionaries TO 'mysql.session'@'localhost'; include/assert.inc [gen_dictionary on an empty table must return NULL] SET @check_expression_result = gen_blocklist('Berlin', 'de_cities', 'us_cities'); include/assert.inc [the result of evaluating 'gen_blocklist('Berlin', 'de_cities', 'us_cities')' must be equal to 'Berlin'] @@ -83,9 +113,30 @@ include/assert.inc [collation of the result of evaluating 'gen_blocklist('Berlin INSERT INTO mysql.masking_dictionaries VALUES('us_cities', 'city1'); INSERT INTO mysql.masking_dictionaries VALUES('us_cities', 'city2'); INSERT INTO mysql.masking_dictionaries VALUES('us_cities', 'city3'); -INSERT INTO mysql.masking_dictionaries VALUES('us_cities', 'city4'); INSERT INTO mysql.masking_dictionaries VALUES('укр_міста', 'місто1'); +include/assert.inc [gen_dictionary on a existing but not flushed dictionary must return NULL] +SELECT masking_dictionaries_flush(); +masking_dictionaries_flush() +1 +include/assert.inc [the number of distinct US city names after the first insert and flush must be 3] +INSERT INTO mysql.masking_dictionaries VALUES('us_cities', 'city4'); INSERT INTO mysql.masking_dictionaries VALUES('укр_міста', 'місто2'); +include/assert.inc [the number of distinct US city names after the second insert but before flush must be 3] +SELECT masking_dictionaries_flush(); +masking_dictionaries_flush() +1 +include/assert.inc [the number of distinct US city names after the second insert and flush must be 4] +RENAME TABLE mysql.masking_dictionaries TO mysql.masking_dictionaries_hidden; +include/assert.inc [the number of distinct US city names after hiding dict table must be 4] +SELECT masking_dictionaries_flush(); +ERROR HY000: Error in command service backend interface, because of : "Table 'mysql.masking_dictionaries' doesn't exist" +include/assert.inc [the number of distinct US city names after dict unsuccessful flush must be 4] +RENAME TABLE mysql.masking_dictionaries_hidden TO mysql.masking_dictionaries; +include/assert.inc [the number of distinct US city names after restoring dict table must be 4] +SELECT masking_dictionaries_flush(); +masking_dictionaries_flush() +1 +include/assert.inc [the number of distinct US city names after restoring dict table and flush must be 4] include/assert.inc [gen_dictionary on a non-existing dictionary must return NULL] SET @check_expression_result = gen_dictionary('us_cities'); include/assert.inc [the result of evaluating 'gen_dictionary('us_cities')' must match the 'city[[:digit:]]{1}' pattern] @@ -117,49 +168,35 @@ include/assert.inc [charset of the result of evaluating 'gen_blocklist('city1', include/assert.inc [collation of the result of evaluating 'gen_blocklist('city1', 'us_cities', 'укр_міста')' must be 'utf8mb4_0900_ai_ci'] include/assert.inc [gen_blocklist when to_dictionary does not exist must return NULL] DELETE FROM mysql.masking_dictionaries; -GRANT MASKING_DICTIONARIES_ADMIN ON *.* TO udftest; +SELECT masking_dictionaries_flush(); +masking_dictionaries_flush() +1 +# # argument nullness checks for functions requiring MASKING_DICTIONARIES_ADMIN SELECT masking_dictionary_term_add(NULL, 'entry'); -ERROR HY000: masking_dictionary_term_add UDF failed; argument 1 cannot be null +ERROR HY000: masking_dictionary_term_add UDF failed; argument 1 cannot be null SELECT masking_dictionary_term_add('single_dict', NULL); -ERROR HY000: masking_dictionary_term_add UDF failed; argument 2 cannot be null +ERROR HY000: masking_dictionary_term_add UDF failed; argument 2 cannot be null SELECT masking_dictionary_term_remove(NULL, 'entry'); -ERROR HY000: masking_dictionary_term_remove UDF failed; argument 1 cannot be null +ERROR HY000: masking_dictionary_term_remove UDF failed; argument 1 cannot be null SELECT masking_dictionary_term_remove('single_dict', NULL); -ERROR HY000: masking_dictionary_term_remove UDF failed; argument 2 cannot be null +ERROR HY000: masking_dictionary_term_remove UDF failed; argument 2 cannot be null SELECT masking_dictionary_remove(NULL); -ERROR HY000: masking_dictionary_remove UDF failed; argument 1 cannot be null +ERROR HY000: masking_dictionary_remove UDF failed; argument 1 cannot be null +# # checks for a user with MASKING_DICTIONARIES_ADMIN privilege using various character sets / collations SET @regular_charset_list = '[ "utf8mb4", "utf8mb3", "utf16", "utf16le", "utf32", "ucs2", "koi8u"]'; SET @special_charset_list = '[ "latin2", "ascii", "binary"]'; SET @charset_list = JSON_MERGE_PRESERVE(@regular_charset_list, @special_charset_list); chacacter set 'utf8mb4' masking_dictionary_term_add checks -SET @check_expression_result = masking_dictionary_term_add('single', CONVERT('term00_юра' USING utf8mb4)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('single', CONVERT('term00_юра' USING utf8mb4))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('single', CONVERT('term00_юра' USING utf8mb4))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('single', CONVERT('term00_юра' USING utf8mb4))' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_term_add("single", CONVERT("term00_юра" USING utf8mb4)) = 1' must succeed] include/assert.inc [second masking_dictionary_term_add must fail] -SET @check_expression_result = masking_dictionary_term_add('dictionary', CONVERT('term10_юра' USING utf8mb4)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term10_юра' USING utf8mb4))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term10_юра' USING utf8mb4))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term10_юра' USING utf8mb4))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('dictionary', CONVERT('term11_юра' USING utf8mb4)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term11_юра' USING utf8mb4))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term11_юра' USING utf8mb4))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term11_юра' USING utf8mb4))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('словник', CONVERT('term20_юра' USING utf8mb4)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term20_юра' USING utf8mb4))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term20_юра' USING utf8mb4))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term20_юра' USING utf8mb4))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('словник', CONVERT('term21_юра' USING utf8mb4)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term21_юра' USING utf8mb4))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term21_юра' USING utf8mb4))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term21_юра' USING utf8mb4))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('словник', CONVERT('term22_юра' USING utf8mb4)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term22_юра' USING utf8mb4))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term22_юра' USING utf8mb4))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term22_юра' USING utf8mb4))' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_term_add("dictionary", CONVERT("term10_юра" USING utf8mb4)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("dictionary", CONVERT("term11_юра" USING utf8mb4)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("словник", CONVERT("term20_юра" USING utf8mb4)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("словник", CONVERT("term21_юра" USING utf8mb4)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("словник", CONVERT("term22_юра" USING utf8mb4)) = 1' must succeed] include/assert.inc [mysql.masking_dictionaries must have 6 records] gen_dictionary: iteration 0 SET @check_expression_result = gen_dictionary('single'); @@ -578,64 +615,25 @@ include/assert.inc [the result of evaluating 'gen_blocklist(CONVERT('term20_юр include/assert.inc [charset of the result of evaluating 'gen_blocklist(CONVERT('term20_юра' USING utf8mb4), 'словник', 'dictionary')' must be 'utf8mb4'] include/assert.inc [collation of the result of evaluating 'gen_blocklist(CONVERT('term20_юра' USING utf8mb4), 'словник', 'dictionary')' must be 'utf8mb4_0900_ai_ci'] masking_dictionary_term_remove checks -SET @check_expression_result = masking_dictionary_term_remove('single', CONVERT('term00_юра' USING utf8mb4)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_remove('single', CONVERT('term00_юра' USING utf8mb4))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_remove('single', CONVERT('term00_юра' USING utf8mb4))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_remove('single', CONVERT('term00_юра' USING utf8mb4))' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_term_remove("single", CONVERT("term00_юра" USING utf8mb4)) = 1' must succeed] include/assert.inc [second masking_dictionary_term_remove must fail] include/assert.inc [mysql.masking_dictionaries must have 5 records] masking_dictionary_remove checks -SET @check_expression_result = masking_dictionary_remove('dictionary'); -include/assert.inc [the result of evaluating 'masking_dictionary_remove('dictionary')' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_remove('dictionary')' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_remove('dictionary')' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_remove("dictionary") = 1' must succeed] include/assert.inc [second masking_dictionary_remove must fail] include/assert.inc [mysql.masking_dictionaries must have 3 records] -SET @check_expression_result = masking_dictionary_remove('словник'); -include/assert.inc [the result of evaluating 'masking_dictionary_remove('словник')' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_remove('словник')' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_remove('словник')' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_remove("словник") = 1' must succeed] include/assert.inc [mysql.masking_dictionaries must have 0 records] ################################ chacacter set 'utf8mb3' masking_dictionary_term_add checks -SET @check_expression_result = masking_dictionary_term_add('single', CONVERT('term00_юра' USING utf8mb3)); -Warnings: -Warning 1287 'utf8mb3' is deprecated and will be removed in a future release. Please use utf8mb4 instead -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('single', CONVERT('term00_юра' USING utf8mb3))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('single', CONVERT('term00_юра' USING utf8mb3))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('single', CONVERT('term00_юра' USING utf8mb3))' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_term_add("single", CONVERT("term00_юра" USING utf8mb3)) = 1' must succeed] include/assert.inc [second masking_dictionary_term_add must fail] -SET @check_expression_result = masking_dictionary_term_add('dictionary', CONVERT('term10_юра' USING utf8mb3)); -Warnings: -Warning 1287 'utf8mb3' is deprecated and will be removed in a future release. Please use utf8mb4 instead -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term10_юра' USING utf8mb3))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term10_юра' USING utf8mb3))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term10_юра' USING utf8mb3))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('dictionary', CONVERT('term11_юра' USING utf8mb3)); -Warnings: -Warning 1287 'utf8mb3' is deprecated and will be removed in a future release. Please use utf8mb4 instead -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term11_юра' USING utf8mb3))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term11_юра' USING utf8mb3))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term11_юра' USING utf8mb3))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('словник', CONVERT('term20_юра' USING utf8mb3)); -Warnings: -Warning 1287 'utf8mb3' is deprecated and will be removed in a future release. Please use utf8mb4 instead -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term20_юра' USING utf8mb3))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term20_юра' USING utf8mb3))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term20_юра' USING utf8mb3))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('словник', CONVERT('term21_юра' USING utf8mb3)); -Warnings: -Warning 1287 'utf8mb3' is deprecated and will be removed in a future release. Please use utf8mb4 instead -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term21_юра' USING utf8mb3))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term21_юра' USING utf8mb3))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term21_юра' USING utf8mb3))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('словник', CONVERT('term22_юра' USING utf8mb3)); -Warnings: -Warning 1287 'utf8mb3' is deprecated and will be removed in a future release. Please use utf8mb4 instead -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term22_юра' USING utf8mb3))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term22_юра' USING utf8mb3))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term22_юра' USING utf8mb3))' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_term_add("dictionary", CONVERT("term10_юра" USING utf8mb3)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("dictionary", CONVERT("term11_юра" USING utf8mb3)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("словник", CONVERT("term20_юра" USING utf8mb3)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("словник", CONVERT("term21_юра" USING utf8mb3)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("словник", CONVERT("term22_юра" USING utf8mb3)) = 1' must succeed] include/assert.inc [mysql.masking_dictionaries must have 6 records] gen_dictionary: iteration 0 SET @check_expression_result = gen_dictionary('single'); @@ -1150,54 +1148,25 @@ include/assert.inc [the result of evaluating 'gen_blocklist(CONVERT('term20_юр include/assert.inc [charset of the result of evaluating 'gen_blocklist(CONVERT('term20_юра' USING utf8mb3), 'словник', 'dictionary')' must be 'utf8mb3'] include/assert.inc [collation of the result of evaluating 'gen_blocklist(CONVERT('term20_юра' USING utf8mb3), 'словник', 'dictionary')' must be 'utf8mb3_general_ci'] masking_dictionary_term_remove checks -SET @check_expression_result = masking_dictionary_term_remove('single', CONVERT('term00_юра' USING utf8mb3)); -Warnings: -Warning 1287 'utf8mb3' is deprecated and will be removed in a future release. Please use utf8mb4 instead -include/assert.inc [the result of evaluating 'masking_dictionary_term_remove('single', CONVERT('term00_юра' USING utf8mb3))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_remove('single', CONVERT('term00_юра' USING utf8mb3))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_remove('single', CONVERT('term00_юра' USING utf8mb3))' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_term_remove("single", CONVERT("term00_юра" USING utf8mb3)) = 1' must succeed] include/assert.inc [second masking_dictionary_term_remove must fail] include/assert.inc [mysql.masking_dictionaries must have 5 records] masking_dictionary_remove checks -SET @check_expression_result = masking_dictionary_remove('dictionary'); -include/assert.inc [the result of evaluating 'masking_dictionary_remove('dictionary')' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_remove('dictionary')' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_remove('dictionary')' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_remove("dictionary") = 1' must succeed] include/assert.inc [second masking_dictionary_remove must fail] include/assert.inc [mysql.masking_dictionaries must have 3 records] -SET @check_expression_result = masking_dictionary_remove('словник'); -include/assert.inc [the result of evaluating 'masking_dictionary_remove('словник')' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_remove('словник')' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_remove('словник')' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_remove("словник") = 1' must succeed] include/assert.inc [mysql.masking_dictionaries must have 0 records] ################################ chacacter set 'utf16' masking_dictionary_term_add checks -SET @check_expression_result = masking_dictionary_term_add('single', CONVERT('term00_юра' USING utf16)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('single', CONVERT('term00_юра' USING utf16))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('single', CONVERT('term00_юра' USING utf16))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('single', CONVERT('term00_юра' USING utf16))' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_term_add("single", CONVERT("term00_юра" USING utf16)) = 1' must succeed] include/assert.inc [second masking_dictionary_term_add must fail] -SET @check_expression_result = masking_dictionary_term_add('dictionary', CONVERT('term10_юра' USING utf16)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term10_юра' USING utf16))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term10_юра' USING utf16))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term10_юра' USING utf16))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('dictionary', CONVERT('term11_юра' USING utf16)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term11_юра' USING utf16))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term11_юра' USING utf16))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term11_юра' USING utf16))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('словник', CONVERT('term20_юра' USING utf16)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term20_юра' USING utf16))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term20_юра' USING utf16))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term20_юра' USING utf16))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('словник', CONVERT('term21_юра' USING utf16)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term21_юра' USING utf16))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term21_юра' USING utf16))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term21_юра' USING utf16))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('словник', CONVERT('term22_юра' USING utf16)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term22_юра' USING utf16))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term22_юра' USING utf16))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term22_юра' USING utf16))' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_term_add("dictionary", CONVERT("term10_юра" USING utf16)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("dictionary", CONVERT("term11_юра" USING utf16)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("словник", CONVERT("term20_юра" USING utf16)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("словник", CONVERT("term21_юра" USING utf16)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("словник", CONVERT("term22_юра" USING utf16)) = 1' must succeed] include/assert.inc [mysql.masking_dictionaries must have 6 records] gen_dictionary: iteration 0 SET @check_expression_result = gen_dictionary('single'); @@ -1616,52 +1585,25 @@ include/assert.inc [the result of evaluating 'gen_blocklist(CONVERT('term20_юр include/assert.inc [charset of the result of evaluating 'gen_blocklist(CONVERT('term20_юра' USING utf16), 'словник', 'dictionary')' must be 'utf16'] include/assert.inc [collation of the result of evaluating 'gen_blocklist(CONVERT('term20_юра' USING utf16), 'словник', 'dictionary')' must be 'utf16_general_ci'] masking_dictionary_term_remove checks -SET @check_expression_result = masking_dictionary_term_remove('single', CONVERT('term00_юра' USING utf16)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_remove('single', CONVERT('term00_юра' USING utf16))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_remove('single', CONVERT('term00_юра' USING utf16))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_remove('single', CONVERT('term00_юра' USING utf16))' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_term_remove("single", CONVERT("term00_юра" USING utf16)) = 1' must succeed] include/assert.inc [second masking_dictionary_term_remove must fail] include/assert.inc [mysql.masking_dictionaries must have 5 records] masking_dictionary_remove checks -SET @check_expression_result = masking_dictionary_remove('dictionary'); -include/assert.inc [the result of evaluating 'masking_dictionary_remove('dictionary')' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_remove('dictionary')' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_remove('dictionary')' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_remove("dictionary") = 1' must succeed] include/assert.inc [second masking_dictionary_remove must fail] include/assert.inc [mysql.masking_dictionaries must have 3 records] -SET @check_expression_result = masking_dictionary_remove('словник'); -include/assert.inc [the result of evaluating 'masking_dictionary_remove('словник')' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_remove('словник')' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_remove('словник')' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_remove("словник") = 1' must succeed] include/assert.inc [mysql.masking_dictionaries must have 0 records] ################################ chacacter set 'utf16le' masking_dictionary_term_add checks -SET @check_expression_result = masking_dictionary_term_add('single', CONVERT('term00_юра' USING utf16le)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('single', CONVERT('term00_юра' USING utf16le))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('single', CONVERT('term00_юра' USING utf16le))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('single', CONVERT('term00_юра' USING utf16le))' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_term_add("single", CONVERT("term00_юра" USING utf16le)) = 1' must succeed] include/assert.inc [second masking_dictionary_term_add must fail] -SET @check_expression_result = masking_dictionary_term_add('dictionary', CONVERT('term10_юра' USING utf16le)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term10_юра' USING utf16le))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term10_юра' USING utf16le))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term10_юра' USING utf16le))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('dictionary', CONVERT('term11_юра' USING utf16le)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term11_юра' USING utf16le))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term11_юра' USING utf16le))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term11_юра' USING utf16le))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('словник', CONVERT('term20_юра' USING utf16le)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term20_юра' USING utf16le))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term20_юра' USING utf16le))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term20_юра' USING utf16le))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('словник', CONVERT('term21_юра' USING utf16le)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term21_юра' USING utf16le))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term21_юра' USING utf16le))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term21_юра' USING utf16le))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('словник', CONVERT('term22_юра' USING utf16le)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term22_юра' USING utf16le))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term22_юра' USING utf16le))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term22_юра' USING utf16le))' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_term_add("dictionary", CONVERT("term10_юра" USING utf16le)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("dictionary", CONVERT("term11_юра" USING utf16le)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("словник", CONVERT("term20_юра" USING utf16le)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("словник", CONVERT("term21_юра" USING utf16le)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("словник", CONVERT("term22_юра" USING utf16le)) = 1' must succeed] include/assert.inc [mysql.masking_dictionaries must have 6 records] gen_dictionary: iteration 0 SET @check_expression_result = gen_dictionary('single'); @@ -2080,52 +2022,25 @@ include/assert.inc [the result of evaluating 'gen_blocklist(CONVERT('term20_юр include/assert.inc [charset of the result of evaluating 'gen_blocklist(CONVERT('term20_юра' USING utf16le), 'словник', 'dictionary')' must be 'utf16le'] include/assert.inc [collation of the result of evaluating 'gen_blocklist(CONVERT('term20_юра' USING utf16le), 'словник', 'dictionary')' must be 'utf16le_general_ci'] masking_dictionary_term_remove checks -SET @check_expression_result = masking_dictionary_term_remove('single', CONVERT('term00_юра' USING utf16le)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_remove('single', CONVERT('term00_юра' USING utf16le))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_remove('single', CONVERT('term00_юра' USING utf16le))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_remove('single', CONVERT('term00_юра' USING utf16le))' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_term_remove("single", CONVERT("term00_юра" USING utf16le)) = 1' must succeed] include/assert.inc [second masking_dictionary_term_remove must fail] include/assert.inc [mysql.masking_dictionaries must have 5 records] masking_dictionary_remove checks -SET @check_expression_result = masking_dictionary_remove('dictionary'); -include/assert.inc [the result of evaluating 'masking_dictionary_remove('dictionary')' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_remove('dictionary')' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_remove('dictionary')' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_remove("dictionary") = 1' must succeed] include/assert.inc [second masking_dictionary_remove must fail] include/assert.inc [mysql.masking_dictionaries must have 3 records] -SET @check_expression_result = masking_dictionary_remove('словник'); -include/assert.inc [the result of evaluating 'masking_dictionary_remove('словник')' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_remove('словник')' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_remove('словник')' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_remove("словник") = 1' must succeed] include/assert.inc [mysql.masking_dictionaries must have 0 records] ################################ chacacter set 'utf32' masking_dictionary_term_add checks -SET @check_expression_result = masking_dictionary_term_add('single', CONVERT('term00_юра' USING utf32)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('single', CONVERT('term00_юра' USING utf32))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('single', CONVERT('term00_юра' USING utf32))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('single', CONVERT('term00_юра' USING utf32))' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_term_add("single", CONVERT("term00_юра" USING utf32)) = 1' must succeed] include/assert.inc [second masking_dictionary_term_add must fail] -SET @check_expression_result = masking_dictionary_term_add('dictionary', CONVERT('term10_юра' USING utf32)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term10_юра' USING utf32))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term10_юра' USING utf32))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term10_юра' USING utf32))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('dictionary', CONVERT('term11_юра' USING utf32)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term11_юра' USING utf32))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term11_юра' USING utf32))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term11_юра' USING utf32))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('словник', CONVERT('term20_юра' USING utf32)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term20_юра' USING utf32))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term20_юра' USING utf32))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term20_юра' USING utf32))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('словник', CONVERT('term21_юра' USING utf32)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term21_юра' USING utf32))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term21_юра' USING utf32))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term21_юра' USING utf32))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('словник', CONVERT('term22_юра' USING utf32)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term22_юра' USING utf32))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term22_юра' USING utf32))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term22_юра' USING utf32))' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_term_add("dictionary", CONVERT("term10_юра" USING utf32)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("dictionary", CONVERT("term11_юра" USING utf32)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("словник", CONVERT("term20_юра" USING utf32)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("словник", CONVERT("term21_юра" USING utf32)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("словник", CONVERT("term22_юра" USING utf32)) = 1' must succeed] include/assert.inc [mysql.masking_dictionaries must have 6 records] gen_dictionary: iteration 0 SET @check_expression_result = gen_dictionary('single'); @@ -2544,64 +2459,25 @@ include/assert.inc [the result of evaluating 'gen_blocklist(CONVERT('term20_юр include/assert.inc [charset of the result of evaluating 'gen_blocklist(CONVERT('term20_юра' USING utf32), 'словник', 'dictionary')' must be 'utf32'] include/assert.inc [collation of the result of evaluating 'gen_blocklist(CONVERT('term20_юра' USING utf32), 'словник', 'dictionary')' must be 'utf32_general_ci'] masking_dictionary_term_remove checks -SET @check_expression_result = masking_dictionary_term_remove('single', CONVERT('term00_юра' USING utf32)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_remove('single', CONVERT('term00_юра' USING utf32))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_remove('single', CONVERT('term00_юра' USING utf32))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_remove('single', CONVERT('term00_юра' USING utf32))' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_term_remove("single", CONVERT("term00_юра" USING utf32)) = 1' must succeed] include/assert.inc [second masking_dictionary_term_remove must fail] include/assert.inc [mysql.masking_dictionaries must have 5 records] masking_dictionary_remove checks -SET @check_expression_result = masking_dictionary_remove('dictionary'); -include/assert.inc [the result of evaluating 'masking_dictionary_remove('dictionary')' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_remove('dictionary')' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_remove('dictionary')' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_remove("dictionary") = 1' must succeed] include/assert.inc [second masking_dictionary_remove must fail] include/assert.inc [mysql.masking_dictionaries must have 3 records] -SET @check_expression_result = masking_dictionary_remove('словник'); -include/assert.inc [the result of evaluating 'masking_dictionary_remove('словник')' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_remove('словник')' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_remove('словник')' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_remove("словник") = 1' must succeed] include/assert.inc [mysql.masking_dictionaries must have 0 records] ################################ chacacter set 'ucs2' masking_dictionary_term_add checks -SET @check_expression_result = masking_dictionary_term_add('single', CONVERT('term00_юра' USING ucs2)); -Warnings: -Warning 1287 'ucs2' is deprecated and will be removed in a future release. Please use utf8mb4 instead -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('single', CONVERT('term00_юра' USING ucs2))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('single', CONVERT('term00_юра' USING ucs2))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('single', CONVERT('term00_юра' USING ucs2))' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_term_add("single", CONVERT("term00_юра" USING ucs2)) = 1' must succeed] include/assert.inc [second masking_dictionary_term_add must fail] -SET @check_expression_result = masking_dictionary_term_add('dictionary', CONVERT('term10_юра' USING ucs2)); -Warnings: -Warning 1287 'ucs2' is deprecated and will be removed in a future release. Please use utf8mb4 instead -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term10_юра' USING ucs2))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term10_юра' USING ucs2))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term10_юра' USING ucs2))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('dictionary', CONVERT('term11_юра' USING ucs2)); -Warnings: -Warning 1287 'ucs2' is deprecated and will be removed in a future release. Please use utf8mb4 instead -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term11_юра' USING ucs2))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term11_юра' USING ucs2))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term11_юра' USING ucs2))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('словник', CONVERT('term20_юра' USING ucs2)); -Warnings: -Warning 1287 'ucs2' is deprecated and will be removed in a future release. Please use utf8mb4 instead -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term20_юра' USING ucs2))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term20_юра' USING ucs2))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term20_юра' USING ucs2))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('словник', CONVERT('term21_юра' USING ucs2)); -Warnings: -Warning 1287 'ucs2' is deprecated and will be removed in a future release. Please use utf8mb4 instead -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term21_юра' USING ucs2))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term21_юра' USING ucs2))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term21_юра' USING ucs2))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('словник', CONVERT('term22_юра' USING ucs2)); -Warnings: -Warning 1287 'ucs2' is deprecated and will be removed in a future release. Please use utf8mb4 instead -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term22_юра' USING ucs2))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term22_юра' USING ucs2))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term22_юра' USING ucs2))' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_term_add("dictionary", CONVERT("term10_юра" USING ucs2)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("dictionary", CONVERT("term11_юра" USING ucs2)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("словник", CONVERT("term20_юра" USING ucs2)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("словник", CONVERT("term21_юра" USING ucs2)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("словник", CONVERT("term22_юра" USING ucs2)) = 1' must succeed] include/assert.inc [mysql.masking_dictionaries must have 6 records] gen_dictionary: iteration 0 SET @check_expression_result = gen_dictionary('single'); @@ -3116,54 +2992,25 @@ include/assert.inc [the result of evaluating 'gen_blocklist(CONVERT('term20_юр include/assert.inc [charset of the result of evaluating 'gen_blocklist(CONVERT('term20_юра' USING ucs2), 'словник', 'dictionary')' must be 'ucs2'] include/assert.inc [collation of the result of evaluating 'gen_blocklist(CONVERT('term20_юра' USING ucs2), 'словник', 'dictionary')' must be 'ucs2_general_ci'] masking_dictionary_term_remove checks -SET @check_expression_result = masking_dictionary_term_remove('single', CONVERT('term00_юра' USING ucs2)); -Warnings: -Warning 1287 'ucs2' is deprecated and will be removed in a future release. Please use utf8mb4 instead -include/assert.inc [the result of evaluating 'masking_dictionary_term_remove('single', CONVERT('term00_юра' USING ucs2))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_remove('single', CONVERT('term00_юра' USING ucs2))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_remove('single', CONVERT('term00_юра' USING ucs2))' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_term_remove("single", CONVERT("term00_юра" USING ucs2)) = 1' must succeed] include/assert.inc [second masking_dictionary_term_remove must fail] include/assert.inc [mysql.masking_dictionaries must have 5 records] masking_dictionary_remove checks -SET @check_expression_result = masking_dictionary_remove('dictionary'); -include/assert.inc [the result of evaluating 'masking_dictionary_remove('dictionary')' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_remove('dictionary')' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_remove('dictionary')' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_remove("dictionary") = 1' must succeed] include/assert.inc [second masking_dictionary_remove must fail] include/assert.inc [mysql.masking_dictionaries must have 3 records] -SET @check_expression_result = masking_dictionary_remove('словник'); -include/assert.inc [the result of evaluating 'masking_dictionary_remove('словник')' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_remove('словник')' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_remove('словник')' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_remove("словник") = 1' must succeed] include/assert.inc [mysql.masking_dictionaries must have 0 records] ################################ chacacter set 'koi8u' masking_dictionary_term_add checks -SET @check_expression_result = masking_dictionary_term_add('single', CONVERT('term00_юра' USING koi8u)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('single', CONVERT('term00_юра' USING koi8u))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('single', CONVERT('term00_юра' USING koi8u))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('single', CONVERT('term00_юра' USING koi8u))' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_term_add("single", CONVERT("term00_юра" USING koi8u)) = 1' must succeed] include/assert.inc [second masking_dictionary_term_add must fail] -SET @check_expression_result = masking_dictionary_term_add('dictionary', CONVERT('term10_юра' USING koi8u)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term10_юра' USING koi8u))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term10_юра' USING koi8u))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term10_юра' USING koi8u))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('dictionary', CONVERT('term11_юра' USING koi8u)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term11_юра' USING koi8u))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term11_юра' USING koi8u))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term11_юра' USING koi8u))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('словник', CONVERT('term20_юра' USING koi8u)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term20_юра' USING koi8u))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term20_юра' USING koi8u))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term20_юра' USING koi8u))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('словник', CONVERT('term21_юра' USING koi8u)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term21_юра' USING koi8u))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term21_юра' USING koi8u))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term21_юра' USING koi8u))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('словник', CONVERT('term22_юра' USING koi8u)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term22_юра' USING koi8u))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term22_юра' USING koi8u))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term22_юра' USING koi8u))' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_term_add("dictionary", CONVERT("term10_юра" USING koi8u)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("dictionary", CONVERT("term11_юра" USING koi8u)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("словник", CONVERT("term20_юра" USING koi8u)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("словник", CONVERT("term21_юра" USING koi8u)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("словник", CONVERT("term22_юра" USING koi8u)) = 1' must succeed] include/assert.inc [mysql.masking_dictionaries must have 6 records] gen_dictionary: iteration 0 SET @check_expression_result = gen_dictionary('single'); @@ -3582,52 +3429,25 @@ include/assert.inc [the result of evaluating 'gen_blocklist(CONVERT('term20_юр include/assert.inc [charset of the result of evaluating 'gen_blocklist(CONVERT('term20_юра' USING koi8u), 'словник', 'dictionary')' must be 'koi8u'] include/assert.inc [collation of the result of evaluating 'gen_blocklist(CONVERT('term20_юра' USING koi8u), 'словник', 'dictionary')' must be 'koi8u_general_ci'] masking_dictionary_term_remove checks -SET @check_expression_result = masking_dictionary_term_remove('single', CONVERT('term00_юра' USING koi8u)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_remove('single', CONVERT('term00_юра' USING koi8u))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_remove('single', CONVERT('term00_юра' USING koi8u))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_remove('single', CONVERT('term00_юра' USING koi8u))' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_term_remove("single", CONVERT("term00_юра" USING koi8u)) = 1' must succeed] include/assert.inc [second masking_dictionary_term_remove must fail] include/assert.inc [mysql.masking_dictionaries must have 5 records] masking_dictionary_remove checks -SET @check_expression_result = masking_dictionary_remove('dictionary'); -include/assert.inc [the result of evaluating 'masking_dictionary_remove('dictionary')' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_remove('dictionary')' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_remove('dictionary')' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_remove("dictionary") = 1' must succeed] include/assert.inc [second masking_dictionary_remove must fail] include/assert.inc [mysql.masking_dictionaries must have 3 records] -SET @check_expression_result = masking_dictionary_remove('словник'); -include/assert.inc [the result of evaluating 'masking_dictionary_remove('словник')' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_remove('словник')' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_remove('словник')' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_remove("словник") = 1' must succeed] include/assert.inc [mysql.masking_dictionaries must have 0 records] ################################ chacacter set 'latin2' masking_dictionary_term_add checks -SET @check_expression_result = masking_dictionary_term_add('single', CONVERT('term00_yura' USING latin2)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('single', CONVERT('term00_yura' USING latin2))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('single', CONVERT('term00_yura' USING latin2))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('single', CONVERT('term00_yura' USING latin2))' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_term_add("single", CONVERT("term00_yura" USING latin2)) = 1' must succeed] include/assert.inc [second masking_dictionary_term_add must fail] -SET @check_expression_result = masking_dictionary_term_add('dictionary', CONVERT('term10_yura' USING latin2)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term10_yura' USING latin2))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term10_yura' USING latin2))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term10_yura' USING latin2))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('dictionary', CONVERT('term11_yura' USING latin2)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term11_yura' USING latin2))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term11_yura' USING latin2))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term11_yura' USING latin2))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('словник', CONVERT('term20_yura' USING latin2)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term20_yura' USING latin2))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term20_yura' USING latin2))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term20_yura' USING latin2))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('словник', CONVERT('term21_yura' USING latin2)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term21_yura' USING latin2))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term21_yura' USING latin2))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term21_yura' USING latin2))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('словник', CONVERT('term22_yura' USING latin2)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term22_yura' USING latin2))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term22_yura' USING latin2))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term22_yura' USING latin2))' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_term_add("dictionary", CONVERT("term10_yura" USING latin2)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("dictionary", CONVERT("term11_yura" USING latin2)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("словник", CONVERT("term20_yura" USING latin2)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("словник", CONVERT("term21_yura" USING latin2)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("словник", CONVERT("term22_yura" USING latin2)) = 1' must succeed] include/assert.inc [mysql.masking_dictionaries must have 6 records] gen_dictionary: iteration 0 SET @check_expression_result = gen_dictionary('single'); @@ -4046,52 +3866,25 @@ include/assert.inc [the result of evaluating 'gen_blocklist(CONVERT('term20_yura include/assert.inc [charset of the result of evaluating 'gen_blocklist(CONVERT('term20_yura' USING latin2), 'словник', 'dictionary')' must be 'latin2'] include/assert.inc [collation of the result of evaluating 'gen_blocklist(CONVERT('term20_yura' USING latin2), 'словник', 'dictionary')' must be 'latin2_general_ci'] masking_dictionary_term_remove checks -SET @check_expression_result = masking_dictionary_term_remove('single', CONVERT('term00_yura' USING latin2)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_remove('single', CONVERT('term00_yura' USING latin2))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_remove('single', CONVERT('term00_yura' USING latin2))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_remove('single', CONVERT('term00_yura' USING latin2))' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_term_remove("single", CONVERT("term00_yura" USING latin2)) = 1' must succeed] include/assert.inc [second masking_dictionary_term_remove must fail] include/assert.inc [mysql.masking_dictionaries must have 5 records] masking_dictionary_remove checks -SET @check_expression_result = masking_dictionary_remove('dictionary'); -include/assert.inc [the result of evaluating 'masking_dictionary_remove('dictionary')' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_remove('dictionary')' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_remove('dictionary')' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_remove("dictionary") = 1' must succeed] include/assert.inc [second masking_dictionary_remove must fail] include/assert.inc [mysql.masking_dictionaries must have 3 records] -SET @check_expression_result = masking_dictionary_remove('словник'); -include/assert.inc [the result of evaluating 'masking_dictionary_remove('словник')' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_remove('словник')' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_remove('словник')' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_remove("словник") = 1' must succeed] include/assert.inc [mysql.masking_dictionaries must have 0 records] ################################ chacacter set 'ascii' masking_dictionary_term_add checks -SET @check_expression_result = masking_dictionary_term_add('single', CONVERT('term00_yura' USING ascii)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('single', CONVERT('term00_yura' USING ascii))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('single', CONVERT('term00_yura' USING ascii))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('single', CONVERT('term00_yura' USING ascii))' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_term_add("single", CONVERT("term00_yura" USING ascii)) = 1' must succeed] include/assert.inc [second masking_dictionary_term_add must fail] -SET @check_expression_result = masking_dictionary_term_add('dictionary', CONVERT('term10_yura' USING ascii)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term10_yura' USING ascii))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term10_yura' USING ascii))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term10_yura' USING ascii))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('dictionary', CONVERT('term11_yura' USING ascii)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term11_yura' USING ascii))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term11_yura' USING ascii))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term11_yura' USING ascii))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('словник', CONVERT('term20_yura' USING ascii)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term20_yura' USING ascii))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term20_yura' USING ascii))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term20_yura' USING ascii))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('словник', CONVERT('term21_yura' USING ascii)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term21_yura' USING ascii))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term21_yura' USING ascii))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term21_yura' USING ascii))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('словник', CONVERT('term22_yura' USING ascii)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term22_yura' USING ascii))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term22_yura' USING ascii))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term22_yura' USING ascii))' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_term_add("dictionary", CONVERT("term10_yura" USING ascii)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("dictionary", CONVERT("term11_yura" USING ascii)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("словник", CONVERT("term20_yura" USING ascii)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("словник", CONVERT("term21_yura" USING ascii)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("словник", CONVERT("term22_yura" USING ascii)) = 1' must succeed] include/assert.inc [mysql.masking_dictionaries must have 6 records] gen_dictionary: iteration 0 SET @check_expression_result = gen_dictionary('single'); @@ -4510,52 +4303,25 @@ include/assert.inc [the result of evaluating 'gen_blocklist(CONVERT('term20_yura include/assert.inc [charset of the result of evaluating 'gen_blocklist(CONVERT('term20_yura' USING ascii), 'словник', 'dictionary')' must be 'ascii'] include/assert.inc [collation of the result of evaluating 'gen_blocklist(CONVERT('term20_yura' USING ascii), 'словник', 'dictionary')' must be 'ascii_general_ci'] masking_dictionary_term_remove checks -SET @check_expression_result = masking_dictionary_term_remove('single', CONVERT('term00_yura' USING ascii)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_remove('single', CONVERT('term00_yura' USING ascii))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_remove('single', CONVERT('term00_yura' USING ascii))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_remove('single', CONVERT('term00_yura' USING ascii))' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_term_remove("single", CONVERT("term00_yura" USING ascii)) = 1' must succeed] include/assert.inc [second masking_dictionary_term_remove must fail] include/assert.inc [mysql.masking_dictionaries must have 5 records] masking_dictionary_remove checks -SET @check_expression_result = masking_dictionary_remove('dictionary'); -include/assert.inc [the result of evaluating 'masking_dictionary_remove('dictionary')' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_remove('dictionary')' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_remove('dictionary')' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_remove("dictionary") = 1' must succeed] include/assert.inc [second masking_dictionary_remove must fail] include/assert.inc [mysql.masking_dictionaries must have 3 records] -SET @check_expression_result = masking_dictionary_remove('словник'); -include/assert.inc [the result of evaluating 'masking_dictionary_remove('словник')' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_remove('словник')' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_remove('словник')' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_remove("словник") = 1' must succeed] include/assert.inc [mysql.masking_dictionaries must have 0 records] ################################ chacacter set 'binary' masking_dictionary_term_add checks -SET @check_expression_result = masking_dictionary_term_add('single', CONVERT('term00_yura' USING binary)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('single', CONVERT('term00_yura' USING binary))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('single', CONVERT('term00_yura' USING binary))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('single', CONVERT('term00_yura' USING binary))' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_term_add("single", CONVERT("term00_yura" USING binary)) = 1' must succeed] include/assert.inc [second masking_dictionary_term_add must fail] -SET @check_expression_result = masking_dictionary_term_add('dictionary', CONVERT('term10_yura' USING binary)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term10_yura' USING binary))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term10_yura' USING binary))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term10_yura' USING binary))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('dictionary', CONVERT('term11_yura' USING binary)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term11_yura' USING binary))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term11_yura' USING binary))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('dictionary', CONVERT('term11_yura' USING binary))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('словник', CONVERT('term20_yura' USING binary)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term20_yura' USING binary))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term20_yura' USING binary))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term20_yura' USING binary))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('словник', CONVERT('term21_yura' USING binary)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term21_yura' USING binary))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term21_yura' USING binary))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term21_yura' USING binary))' must be 'utf8mb4_0900_ai_ci'] -SET @check_expression_result = masking_dictionary_term_add('словник', CONVERT('term22_yura' USING binary)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term22_yura' USING binary))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term22_yura' USING binary))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_add('словник', CONVERT('term22_yura' USING binary))' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_term_add("dictionary", CONVERT("term10_yura" USING binary)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("dictionary", CONVERT("term11_yura" USING binary)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("словник", CONVERT("term20_yura" USING binary)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("словник", CONVERT("term21_yura" USING binary)) = 1' must succeed] +include/assert.inc ['masking_dictionary_term_add("словник", CONVERT("term22_yura" USING binary)) = 1' must succeed] include/assert.inc [mysql.masking_dictionaries must have 6 records] gen_dictionary: iteration 0 SET @check_expression_result = gen_dictionary('single'); @@ -4974,25 +4740,18 @@ include/assert.inc [the result of evaluating 'gen_blocklist(CONVERT('term20_yura include/assert.inc [charset of the result of evaluating 'gen_blocklist(CONVERT('term20_yura' USING binary), 'словник', 'dictionary')' must be 'binary'] include/assert.inc [collation of the result of evaluating 'gen_blocklist(CONVERT('term20_yura' USING binary), 'словник', 'dictionary')' must be 'binary'] masking_dictionary_term_remove checks -SET @check_expression_result = masking_dictionary_term_remove('single', CONVERT('term00_yura' USING binary)); -include/assert.inc [the result of evaluating 'masking_dictionary_term_remove('single', CONVERT('term00_yura' USING binary))' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_term_remove('single', CONVERT('term00_yura' USING binary))' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_term_remove('single', CONVERT('term00_yura' USING binary))' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_term_remove("single", CONVERT("term00_yura" USING binary)) = 1' must succeed] include/assert.inc [second masking_dictionary_term_remove must fail] include/assert.inc [mysql.masking_dictionaries must have 5 records] masking_dictionary_remove checks -SET @check_expression_result = masking_dictionary_remove('dictionary'); -include/assert.inc [the result of evaluating 'masking_dictionary_remove('dictionary')' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_remove('dictionary')' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_remove('dictionary')' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_remove("dictionary") = 1' must succeed] include/assert.inc [second masking_dictionary_remove must fail] include/assert.inc [mysql.masking_dictionaries must have 3 records] -SET @check_expression_result = masking_dictionary_remove('словник'); -include/assert.inc [the result of evaluating 'masking_dictionary_remove('словник')' must be equal to '1'] -include/assert.inc [charset of the result of evaluating 'masking_dictionary_remove('словник')' must be 'utf8mb4'] -include/assert.inc [collation of the result of evaluating 'masking_dictionary_remove('словник')' must be 'utf8mb4_0900_ai_ci'] +include/assert.inc ['masking_dictionary_remove("словник") = 1' must succeed] include/assert.inc [mysql.masking_dictionaries must have 0 records] ################################ -DROP USER udftest; +DROP USER udftest_unpriv@localhost; +DROP USER udftest_priv@localhost; UNINSTALL COMPONENT 'file://component_masking_functions'; +REVOKE SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries FROM 'mysql.session'@'localhost'; DROP TABLE mysql.masking_dictionaries; diff --git a/mysql-test/suite/component_masking_functions/r/flusher_thread_connection_reuse.result b/mysql-test/suite/component_masking_functions/r/flusher_thread_connection_reuse.result new file mode 100644 index 000000000000..7a98b6b4e643 --- /dev/null +++ b/mysql-test/suite/component_masking_functions/r/flusher_thread_connection_reuse.result @@ -0,0 +1,18 @@ +CREATE TABLE mysql.masking_dictionaries( +Dictionary VARCHAR(256) NOT NULL, +Term VARCHAR(256) NOT NULL, +UNIQUE INDEX dictionary_term_idx (Dictionary, Term) +) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4; +GRANT SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries TO 'mysql.session'@'localhost'; +INSERT INTO mysql.masking_dictionaries VALUES('dict1', 'term1'); +SET GLOBAL debug = '+d,enable_masking_functions_flusher_create_sync,enable_masking_functions_flush_thread_sync,enable_masking_functions_flush_thread_double_pass'; +SET debug_sync = 'masking_functions_after_flusher_create WAIT_FOR masking_functions_after_flusher_create_signal'; +INSTALL COMPONENT 'file://component_masking_functions'; +SET debug_sync = 'now SIGNAL masking_functions_before_cache_reload_signal WAIT_FOR masking_functions_after_cache_reload_signal'; +SET debug_sync = 'now SIGNAL masking_functions_after_flusher_create_signal'; +SET debug_sync = 'now SIGNAL masking_functions_before_cache_reload_signal WAIT_FOR masking_functions_after_cache_reload_signal'; +UNINSTALL COMPONENT 'file://component_masking_functions'; +SET GLOBAL debug = '-d,enable_masking_functions_flusher_create_sync,enable_masking_functions_flush_thread_sync,enable_masking_functions_flush_thread_double_pass'; +REVOKE SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries FROM 'mysql.session'@'localhost'; +DROP TABLE mysql.masking_dictionaries; +SET debug_sync = 'RESET'; diff --git a/mysql-test/suite/component_masking_functions/r/flusher_thread_immediate_restart.result b/mysql-test/suite/component_masking_functions/r/flusher_thread_immediate_restart.result new file mode 100644 index 000000000000..f0065e2b6879 --- /dev/null +++ b/mysql-test/suite/component_masking_functions/r/flusher_thread_immediate_restart.result @@ -0,0 +1,15 @@ +CALL mtr.add_suppression("Flusher thread terminated while waiting for session server"); +CALL mtr.add_suppression("Flusher thread terminated after creating internal connection"); +CALL mtr.add_suppression("Server shutdown requested while attempting to establish flusher thread connection"); +CREATE TABLE mysql.masking_dictionaries( +Dictionary VARCHAR(256) NOT NULL, +Term VARCHAR(256) NOT NULL, +UNIQUE INDEX dictionary_term_idx (Dictionary, Term) +) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4; +GRANT SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries TO 'mysql.session'@'localhost'; +INSERT INTO mysql.masking_dictionaries VALUES('dict1', 'term1'); +INSTALL COMPONENT 'file://component_masking_functions'; +# restart +UNINSTALL COMPONENT 'file://component_masking_functions'; +REVOKE SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries FROM 'mysql.session'@'localhost'; +DROP TABLE mysql.masking_dictionaries; diff --git a/mysql-test/suite/component_masking_functions/r/flusher_thread_suspend_resume.result b/mysql-test/suite/component_masking_functions/r/flusher_thread_suspend_resume.result new file mode 100644 index 000000000000..2b97ca59094a --- /dev/null +++ b/mysql-test/suite/component_masking_functions/r/flusher_thread_suspend_resume.result @@ -0,0 +1,25 @@ +CALL mtr.add_suppression("Exception during reloading dictionary cache"); +SET GLOBAL debug = '+d,enable_masking_functions_flusher_create_sync,enable_masking_functions_flush_thread_sync,enable_masking_functions_flush_thread_double_pass'; +SET debug_sync = 'masking_functions_after_flusher_create WAIT_FOR masking_functions_after_flusher_create_signal'; +INSTALL COMPONENT 'file://component_masking_functions'; +SET debug_sync = 'now SIGNAL masking_functions_before_cache_reload_signal WAIT_FOR masking_functions_after_cache_reload_signal'; +include/assert.inc [Server error log must contain an error record from the Dictionary Flusher thread] +SET debug_sync = 'now SIGNAL masking_functions_after_flusher_create_signal'; +CREATE TABLE mysql.masking_dictionaries( +Dictionary VARCHAR(256) NOT NULL, +Term VARCHAR(256) NOT NULL, +UNIQUE INDEX dictionary_term_idx (Dictionary, Term) +) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4; +GRANT SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries TO 'mysql.session'@'localhost'; +INSERT INTO mysql.masking_dictionaries VALUES('dict1', 'term1'); +include/assert.inc [The very first call to gen_dictionary("dict1") should trigger inplace cache reload and return "term1"] +INSERT INTO mysql.masking_dictionaries VALUES('dict2', 'term2'); +include/assert.inc [The very first call to gen_dictionary("dict2") should not trigger inplace cache reload and return NULL] +SET debug_sync = 'now SIGNAL masking_functions_before_cache_reload_signal WAIT_FOR masking_functions_after_cache_reload_signal'; +include/assert.inc [After resuming flusher thread gen_dictionary("dict1") should still return "term1"] +include/assert.inc [After resuming flusher thread gen_dictionary("dict2") should get new values from the cache updated by the flusher and return "term2"] +UNINSTALL COMPONENT 'file://component_masking_functions'; +SET GLOBAL debug = '-d,enable_masking_functions_flusher_create_sync,enable_masking_functions_flush_thread_sync,enable_masking_functions_flush_thread_double_pass'; +REVOKE SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries FROM 'mysql.session'@'localhost'; +DROP TABLE mysql.masking_dictionaries; +SET debug_sync = 'RESET'; diff --git a/mysql-test/suite/component_masking_functions/r/global_autocommit_off.result b/mysql-test/suite/component_masking_functions/r/global_autocommit_off.result index 136e142c5f2f..e04571f9aa82 100644 --- a/mysql-test/suite/component_masking_functions/r/global_autocommit_off.result +++ b/mysql-test/suite/component_masking_functions/r/global_autocommit_off.result @@ -8,6 +8,7 @@ Dictionary VARCHAR(256) NOT NULL, Term VARCHAR(256) NOT NULL, UNIQUE INDEX dictionary_term_idx (Dictionary, Term) ) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4; +GRANT SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries TO 'mysql.session'@'localhost'; GRANT MASKING_DICTIONARIES_ADMIN ON *.* TO root@localhost; include/assert.inc [adding "entry" to "dict" must succeed] include/assert.inc [mysql.masking_dictionaries must have 1 record] @@ -15,6 +16,7 @@ SET @@global.autocommit = OFF; include/assert.inc [adding "another_entry" to "dict" must succeed] include/assert.inc [mysql.masking_dictionaries must have 2 record] REVOKE MASKING_DICTIONARIES_ADMIN ON *.* FROM root@localhost; +REVOKE SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries FROM 'mysql.session'@'localhost'; DROP TABLE mysql.masking_dictionaries; UNINSTALL COMPONENT 'file://component_masking_functions'; SET @@global.autocommit = @saved_autocommit; diff --git a/mysql-test/suite/component_masking_functions/r/rpl_dictionaries_flush_interval.result b/mysql-test/suite/component_masking_functions/r/rpl_dictionaries_flush_interval.result new file mode 100644 index 000000000000..c311a0c32182 --- /dev/null +++ b/mysql-test/suite/component_masking_functions/r/rpl_dictionaries_flush_interval.result @@ -0,0 +1,81 @@ +include/master-slave.inc +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the connection metadata repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START REPLICA; see the 'START REPLICA Syntax' in the MySQL Manual for more information. +[connection master] +[connection master] +CREATE TABLE mysql.masking_dictionaries( +Dictionary VARCHAR(256) NOT NULL, +Term VARCHAR(256) NOT NULL, +UNIQUE INDEX dictionary_term_idx (Dictionary, Term) +) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4; +SET sql_log_bin = OFF; +GRANT SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries TO 'mysql.session'@'localhost'; +SET sql_log_bin = ON; +include/rpl_sync.inc +[connection slave] +GRANT SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries TO 'mysql.session'@'localhost'; +[connection master] +SET GLOBAL debug ='+d,enable_masking_functions_flush_thread_sync'; +INSTALL COMPONENT 'file://component_masking_functions'; +SET sql_log_bin = OFF; +GRANT MASKING_DICTIONARIES_ADMIN ON *.* TO root@localhost; +SET sql_log_bin = ON; +[connection slave] +SET GLOBAL debug ='+d,enable_masking_functions_flush_thread_sync'; +INSTALL COMPONENT 'file://component_masking_functions'; +GRANT MASKING_DICTIONARIES_ADMIN ON *.* TO root@localhost; +[connection master] +include/assert.inc [The very first call to gen_dictionary("dict1") on source must load empty cache and return NULL] +include/rpl_sync.inc +[connection slave] +include/assert.inc [The very first call to gen_dictionary("dict1") on replica must load empty cache and return NULL] +[connection master] +SELECT masking_dictionary_term_add('dict1', 'term1'); +masking_dictionary_term_add('dict1', 'term1') +1 +SELECT masking_dictionary_term_add('dict2', 'term2'); +masking_dictionary_term_add('dict2', 'term2') +1 +include/assert.inc [The call to gen_dictionary("dict1") after inserting terms on source must return "term1"] +include/assert.inc [The call to gen_dictionary("dict2") after inserting terms on source must return "term2"] +include/rpl_sync.inc +[connection slave] +include/assert.inc [The number of dictionary terms on replica after synchronization must be 2] +include/assert.inc [The call to gen_dictionary("dict1") on replica after synchronization but without reloading cache must return NULL] +include/assert.inc [The call to gen_dictionary("dict1") on replica after synchronization but without reloading cache must return NULL] +SELECT masking_dictionaries_flush(); +masking_dictionaries_flush() +1 +include/assert.inc [The call to gen_dictionary("dict1") on replica after synchronization and reloading cache must return "term1"] +include/assert.inc [The call to gen_dictionary("dict1") on replica after synchronization and reloading cache must return "term2"] +[connection master] +INSERT INTO mysql.masking_dictionaries VALUES ('dict3', 'term3'); +include/assert.inc [The call to gen_dictionary("dict3") on source with suspended flusher thread must return NULL] +SET debug_sync = 'now SIGNAL masking_functions_before_cache_reload_signal WAIT_FOR masking_functions_after_cache_reload_signal'; +include/assert.inc [The call to gen_dictionary("dict3") on source after resuming flusher thread must return "term3"] +include/rpl_sync.inc +[connection slave] +include/assert.inc [The number of dictionary terms on replica after the second synchronization must be 3] +include/assert.inc [The call to gen_dictionary("dict3") on replica with suspended flusher thread must return NULL] +SET debug_sync = 'now SIGNAL masking_functions_before_cache_reload_signal WAIT_FOR masking_functions_after_cache_reload_signal'; +include/assert.inc [The call to gen_dictionary("dict3") on replica after resuming flusher thread must return "term3"] +[connection slave] +REVOKE MASKING_DICTIONARIES_ADMIN ON *.* FROM root@localhost; +SET GLOBAL debug ='-d,enable_masking_functions_flush_thread_sync'; +UNINSTALL COMPONENT 'file://component_masking_functions'; +REVOKE SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries FROM 'mysql.session'@'localhost'; +SET debug_sync = 'RESET'; +[connection master] +SET sql_log_bin = OFF; +REVOKE MASKING_DICTIONARIES_ADMIN ON *.* FROM root@localhost; +SET sql_log_bin = ON; +SET GLOBAL debug ='-d,enable_masking_functions_flush_thread_sync'; +UNINSTALL COMPONENT 'file://component_masking_functions'; +SET sql_log_bin = OFF; +REVOKE SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries FROM 'mysql.session'@'localhost'; +SET sql_log_bin = ON; +SET debug_sync = 'RESET'; +DROP TABLE mysql.masking_dictionaries; +include/rpl_sync.inc +include/rpl_end.inc diff --git a/mysql-test/suite/component_masking_functions/r/rpl_function_determinism_mixed.result b/mysql-test/suite/component_masking_functions/r/rpl_function_determinism_mixed.result index 7aaa7c4ebcc7..1a60f50eaf55 100644 --- a/mysql-test/suite/component_masking_functions/r/rpl_function_determinism_mixed.result +++ b/mysql-test/suite/component_masking_functions/r/rpl_function_determinism_mixed.result @@ -331,6 +331,7 @@ Dictionary VARCHAR(256) NOT NULL, Term VARCHAR(256) NOT NULL, UNIQUE INDEX dictionary_term_idx (Dictionary, Term) ) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4; +GRANT SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries TO 'mysql.session'@'localhost'; GRANT MASKING_DICTIONARIES_ADMIN ON *.* TO root@localhost; # masking_dictionary_term_add() SELECT masking_dictionary_term_add('dict1', 'word11'); @@ -386,6 +387,7 @@ include/sync_slave_sql_with_master.inc include/assert.inc [mysql.masking_dictionaries must have no records] [connection master] REVOKE MASKING_DICTIONARIES_ADMIN ON *.* FROM root@localhost; +REVOKE SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries FROM 'mysql.session'@'localhost'; DROP TABLE mysql.masking_dictionaries; [connection slave] UNINSTALL COMPONENT 'file://component_masking_functions'; diff --git a/mysql-test/suite/component_masking_functions/r/rpl_function_determinism_row.result b/mysql-test/suite/component_masking_functions/r/rpl_function_determinism_row.result index 7aaa7c4ebcc7..1a60f50eaf55 100644 --- a/mysql-test/suite/component_masking_functions/r/rpl_function_determinism_row.result +++ b/mysql-test/suite/component_masking_functions/r/rpl_function_determinism_row.result @@ -331,6 +331,7 @@ Dictionary VARCHAR(256) NOT NULL, Term VARCHAR(256) NOT NULL, UNIQUE INDEX dictionary_term_idx (Dictionary, Term) ) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4; +GRANT SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries TO 'mysql.session'@'localhost'; GRANT MASKING_DICTIONARIES_ADMIN ON *.* TO root@localhost; # masking_dictionary_term_add() SELECT masking_dictionary_term_add('dict1', 'word11'); @@ -386,6 +387,7 @@ include/sync_slave_sql_with_master.inc include/assert.inc [mysql.masking_dictionaries must have no records] [connection master] REVOKE MASKING_DICTIONARIES_ADMIN ON *.* FROM root@localhost; +REVOKE SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries FROM 'mysql.session'@'localhost'; DROP TABLE mysql.masking_dictionaries; [connection slave] UNINSTALL COMPONENT 'file://component_masking_functions'; diff --git a/mysql-test/suite/component_masking_functions/r/rpl_function_determinism_statement.result b/mysql-test/suite/component_masking_functions/r/rpl_function_determinism_statement.result index e1177ea929f2..b0f8f0c0a604 100644 --- a/mysql-test/suite/component_masking_functions/r/rpl_function_determinism_statement.result +++ b/mysql-test/suite/component_masking_functions/r/rpl_function_determinism_statement.result @@ -396,6 +396,7 @@ Dictionary VARCHAR(256) NOT NULL, Term VARCHAR(256) NOT NULL, UNIQUE INDEX dictionary_term_idx (Dictionary, Term) ) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4; +GRANT SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries TO 'mysql.session'@'localhost'; GRANT MASKING_DICTIONARIES_ADMIN ON *.* TO root@localhost; # masking_dictionary_term_add() SELECT masking_dictionary_term_add('dict1', 'word11'); @@ -455,6 +456,7 @@ include/sync_slave_sql_with_master.inc include/assert.inc [mysql.masking_dictionaries must have no records] [connection master] REVOKE MASKING_DICTIONARIES_ADMIN ON *.* FROM root@localhost; +REVOKE SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries FROM 'mysql.session'@'localhost'; DROP TABLE mysql.masking_dictionaries; [connection slave] UNINSTALL COMPONENT 'file://component_masking_functions'; diff --git a/mysql-test/suite/component_masking_functions/r/sys_var_dictionaries_flush_interval_seconds_basic.result b/mysql-test/suite/component_masking_functions/r/sys_var_dictionaries_flush_interval_seconds_basic.result new file mode 100644 index 000000000000..8e78bd0b0b77 --- /dev/null +++ b/mysql-test/suite/component_masking_functions/r/sys_var_dictionaries_flush_interval_seconds_basic.result @@ -0,0 +1,37 @@ +CALL mtr.add_suppression("Flusher thread terminated while waiting for session server"); +CALL mtr.add_suppression("Flusher thread terminated after creating internal connection"); +CALL mtr.add_suppression("Server shutdown requested while attempting to establish flusher thread connection"); +INSTALL COMPONENT 'file://component_masking_functions'; +SELECT @@global.masking_functions.dictionaries_flush_interval_seconds; +@@global.masking_functions.dictionaries_flush_interval_seconds +0 +SELECT NAME FROM performance_schema.threads WHERE NAME LIKE "%masking_functions%"; +NAME +SET GLOBAL masking_functions.dictionaries_flush_interval_seconds=100; +ERROR HY000: Variable 'masking_functions.dictionaries_flush_interval_seconds' is a read only variable +SET SESSION masking_functions.dictionaries_flush_interval_seconds=100; +ERROR HY000: Variable 'masking_functions.dictionaries_flush_interval_seconds' is a read only variable +CREATE TABLE mysql.masking_dictionaries( +Dictionary VARCHAR(256) NOT NULL, +Term VARCHAR(256) NOT NULL, +UNIQUE INDEX dictionary_term_idx (Dictionary, Term) +) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4; +GRANT SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries TO 'mysql.session'@'localhost'; +# restart: --masking_functions.dictionaries_flush_interval_seconds=100 +SELECT @@global.masking_functions.dictionaries_flush_interval_seconds; +@@global.masking_functions.dictionaries_flush_interval_seconds +100 +CREATE USER udftest_priv@localhost; +GRANT MASKING_DICTIONARIES_ADMIN ON *.* TO udftest_priv@localhost; +SELECT masking_dictionary_term_add('single_dict_1', 'entry_1'); +masking_dictionary_term_add('single_dict_1', 'entry_1') +1 +SELECT NAME FROM performance_schema.threads WHERE NAME LIKE "%masking_functions%"; +NAME +thread/masking_functions/masking_functions_dict_flusher +REVOKE MASKING_DICTIONARIES_ADMIN ON *.* FROM udftest_priv@localhost; +UNINSTALL COMPONENT 'file://component_masking_functions'; +REVOKE SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries FROM 'mysql.session'@'localhost'; +DROP USER udftest_priv@localhost; +DROP TABLE mysql.masking_dictionaries; +# restart: diff --git a/mysql-test/suite/component_masking_functions/r/sys_var_masking_database.result b/mysql-test/suite/component_masking_functions/r/sys_var_masking_database.result new file mode 100644 index 000000000000..f0edb1a2106b --- /dev/null +++ b/mysql-test/suite/component_masking_functions/r/sys_var_masking_database.result @@ -0,0 +1,28 @@ +INSTALL COMPONENT 'file://component_masking_functions'; +SET GLOBAL masking_functions.masking_database=dict_db; +ERROR HY000: Variable 'masking_functions.masking_database' is a read only variable +SET SESSION masking_functions.masking_database=dict_db; +ERROR HY000: Variable 'masking_functions.masking_database' is a read only variable +# restart: +include/assert_grep.inc [Bad masking_functions.masking_database value] +CREATE DATABASE dict_db; +CREATE TABLE dict_db.masking_dictionaries( +Dictionary VARCHAR(256) NOT NULL, +Term VARCHAR(256) NOT NULL, +UNIQUE INDEX dictionary_term_idx (Dictionary, Term) +) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4; +GRANT SELECT, INSERT, UPDATE, DELETE ON dict_db.masking_dictionaries TO 'mysql.session'@'localhost'; +# restart: +CREATE USER udftest_priv@localhost; +GRANT MASKING_DICTIONARIES_ADMIN ON *.* TO udftest_priv@localhost; +SELECT masking_dictionary_term_add('single_dict', 'entry'); +masking_dictionary_term_add('single_dict', 'entry') +1 +SELECT gen_dictionary('single_dict'); +gen_dictionary('single_dict') +entry +UNINSTALL COMPONENT 'file://component_masking_functions'; +REVOKE SELECT, INSERT, UPDATE, DELETE ON dict_db.masking_dictionaries FROM 'mysql.session'@'localhost'; +DROP TABLE dict_db.masking_dictionaries; +DROP DATABASE dict_db; +DROP USER udftest_priv@localhost; diff --git a/mysql-test/suite/component_masking_functions/t/check_expression.inc b/mysql-test/suite/component_masking_functions/t/check_expression.inc index 5fbc9ed84367..6b517a97395e 100644 --- a/mysql-test/suite/component_masking_functions/t/check_expression.inc +++ b/mysql-test/suite/component_masking_functions/t/check_expression.inc @@ -6,7 +6,7 @@ # $expected_collation - the expected collation of the result (optional) # $hide_value - if non-zero, then the $value will be hidden from the assertion statements # -# Only one of the $regexp / $value should be specified - if both, $regexp has precedence +# Only one of the $regexp / $value should be specified - if both, $value has precedence # if $expected_collation is not specified, then the default one for the given $expected_charset will be assumed eval SET @check_expression_result = $expression; diff --git a/mysql-test/suite/component_masking_functions/t/dictionary_ai_ci_lookups.test b/mysql-test/suite/component_masking_functions/t/dictionary_ai_ci_lookups.test new file mode 100644 index 000000000000..8b7610534b8c --- /dev/null +++ b/mysql-test/suite/component_masking_functions/t/dictionary_ai_ci_lookups.test @@ -0,0 +1,113 @@ +--source include/have_masking_functions_component.inc + +--echo # +--echo # Checking for proper case-insensitive 'utf8mb4_0900_ai_ci' string +--echo # comparison in Masking Function dictionary operations + +--source include/count_sessions.inc + +INSTALL COMPONENT 'file://component_masking_functions'; + +CREATE TABLE mysql.masking_dictionaries( + Dictionary VARCHAR(256) NOT NULL, + Term VARCHAR(256) NOT NULL, + UNIQUE INDEX dictionary_term_idx (Dictionary, Term) +) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4; +GRANT SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries TO 'mysql.session'@'localhost'; + +CREATE USER udftest_priv@localhost; +GRANT MASKING_DICTIONARIES_ADMIN ON *.* TO udftest_priv@localhost; +--connect(con_priv,localhost,udftest_priv,,) + +--echo # +--echo # Polulating dictionary 'dict' +--let $assert_cond = masking_dictionary_term_add("dict", "gross") = 1 +--let $assert_text = The very first masking_dictionary_term_add() must succeed +--source include/assert.inc + + +--echo # +--echo # Checking case-sensitivity and accents in dictionary terms +--let $assert_cond = masking_dictionary_term_add("dict", "Gross") = 0 +--let $assert_text = Terms with differences in case must be considered equal +--source include/assert.inc + +--let $assert_cond = masking_dictionary_term_add("dict", "gröss") = 0 +--let $assert_text = Terms with differences in accent must be considered equal +--source include/assert.inc + +--let $assert_cond = masking_dictionary_term_add("dict", "grÖss") = 0 +--let $assert_text = Terms with differences in both case and accent must be considered equal +--source include/assert.inc + +--let $assert_cond = masking_dictionary_term_add("dict", "groß") = 0 +--let $assert_text = Terms with German Eszett and "ss" must be considered equal +--source include/assert.inc + +--let $assert_cond = masking_dictionary_term_add("dict", "groẞ") = 0 +--let $assert_text = Terms with German Capital Eszett and "ss" must be considered equal +--source include/assert.inc + + +--echo # +--echo # Checking case-sensitivity and accents in dictionary names +--let $assert_cond = masking_dictionary_term_add("Dict", "gross") = 0 +--let $assert_text = Dictionary names with differences in case must be considered equal +--source include/assert.inc + +--let $assert_cond = masking_dictionary_term_add("dïct", "gross") = 0 +--let $assert_text = Dictionary names with differences in accent must be considered equal +--source include/assert.inc + +--let $assert_cond = masking_dictionary_term_add("dÏct", "gross") = 0 +--let $assert_text = Dictionary names with differences in both case and accent must be considered equal +--source include/assert.inc + + +--echo # +--echo # Polulating dictionary 'dummy' (will be used for substitutions) +--let $assert_cond = masking_dictionary_term_add("dummy", "klein") = 1 +--let $assert_text = The second meaningful call to masking_dictionary_term_add() must succeed +--source include/assert.inc + +--echo # +--echo # Checking Dictionary Cache case-insensitive accent-aware lookups via gen_blocklist() +--let $assert_cond = gen_blocklist("gross", "dict", "dummy") = "klein" +--let $assert_text = Original term must be substituted +--source include/assert.inc + +--let $assert_cond = gen_blocklist("Gross", "dict", "dummy") = "klein" +--let $assert_text = Term with changed case must be substituted +--source include/assert.inc + +--let $assert_cond = gen_blocklist("gröss", "dict", "dummy") = "klein" +--let $assert_text = Term with changed accent must be substituted +--source include/assert.inc + +--let $assert_cond = gen_blocklist("grÖss", "dict", "dummy") = "klein" +--let $assert_text = Term with both changed case and changed accent must be substituted +--source include/assert.inc + +--let $assert_cond = gen_blocklist("groß", "dict", "dummy") = "klein" +--let $assert_text = Terms with German Eszett must be substituted +--source include/assert.inc + +--let $assert_cond = gen_blocklist("groẞ", "dict", "dummy") = "klein" +--let $assert_text = Terms with German Capital Eszett must be substituted +--source include/assert.inc + +--let $assert_cond = gen_blocklist("grosss", "dict", "dummy") = "grosss" +--let $assert_text = Terms with actual differences must not be substituted +--source include/assert.inc + +--disconnect con_priv +--connection default + +DROP USER udftest_priv@localhost; + +UNINSTALL COMPONENT 'file://component_masking_functions'; + +REVOKE SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries FROM 'mysql.session'@'localhost'; +DROP TABLE mysql.masking_dictionaries; + +--source include/wait_until_count_sessions.inc diff --git a/mysql-test/suite/component_masking_functions/t/dictionary_operations.test b/mysql-test/suite/component_masking_functions/t/dictionary_operations.test index db73a98b9ab7..0a31d2c52a1c 100644 --- a/mysql-test/suite/component_masking_functions/t/dictionary_operations.test +++ b/mysql-test/suite/component_masking_functions/t/dictionary_operations.test @@ -7,13 +7,26 @@ --echo # * masking_dictionary_term_add --echo # * masking_dictionary_term_remove --echo # * masking_dictionary_remove +--echo # * masking_dictionaries_flush --echo # --source include/count_sessions.inc INSTALL COMPONENT 'file://component_masking_functions'; ---echo # arity checks +--echo # +--echo # Create users with and without MASKING_DICTIONARIES_ADMIN privilege +CREATE USER udftest_unpriv@localhost; +CREATE USER udftest_priv@localhost; +GRANT MASKING_DICTIONARIES_ADMIN ON *.* TO udftest_priv@localhost; +--connect(con_unpriv,localhost,udftest_unpriv,,) +--connect(con_priv,localhost,udftest_priv,,) +--connection default + +--echo # +--echo # arity checks, run for privileged user +--connection con_priv + --error ER_CANT_INITIALIZE_UDF SELECT gen_dictionary(); --error ER_CANT_INITIALIZE_UDF @@ -47,8 +60,12 @@ SELECT masking_dictionary_remove(); --error ER_CANT_INITIALIZE_UDF SELECT masking_dictionary_remove('', ''); +--error ER_CANT_INITIALIZE_UDF +SELECT masking_dictionaries_flush(''); +--echo # --echo # argument nullness checks for functions not requiring MASKING_DICTIONARIES_ADMIN +--connection con_unpriv --error ER_UDF_ERROR SELECT gen_dictionary(NULL); @@ -58,25 +75,45 @@ SELECT gen_blocklist('Berlin', NULL, 'us_cities'); SELECT gen_blocklist('Berlin', 'de_cities', NULL); +--echo # --echo # checking the case when there is no mysql.masking_dictionaries table ---error ER_COMMAND_SERVICE_BACKEND_FAILED -SELECT gen_blocklist('Berlin', 'de_cities', 'us_cities'); +--connection default +# here CREATE is needed to grant privileges to a not-yet-existing table +GRANT CREATE, SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries TO 'mysql.session'@'localhost'; +--connection con_unpriv --error ER_COMMAND_SERVICE_BACKEND_FAILED SELECT gen_dictionary('us_cities'); +--connection con_priv +--error ER_COMMAND_SERVICE_BACKEND_FAILED +SELECT masking_dictionaries_flush(); +--error ER_COMMAND_SERVICE_BACKEND_FAILED +SELECT masking_dictionary_term_add('single_dict', 'entry'); + +--connection default +REVOKE CREATE, SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries FROM 'mysql.session'@'localhost'; +--connection con_unpriv + + +--echo # --echo # NULL for NULL checks --let $assert_cond = gen_blocklist(NULL, "de_cities", "us_cities") IS NULL --let $assert_text = gen_blocklist() for the NULL primary argument should return NULL --source include/assert.inc +--echo # --echo # checking the case when mysql.masking_dictionaries has invalid structure +--connection default CREATE TABLE mysql.masking_dictionaries( Dictionary VARCHAR(256) NOT NULL, UNIQUE INDEX dictionary_term_idx (Dictionary) ) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4; +GRANT SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries TO 'mysql.session'@'localhost'; + +--connection con_unpriv --error ER_COMMAND_SERVICE_BACKEND_FAILED SELECT gen_blocklist('Berlin', 'de_cities', 'us_cities'); @@ -84,13 +121,13 @@ SELECT gen_blocklist('Berlin', 'de_cities', 'us_cities'); --error ER_COMMAND_SERVICE_BACKEND_FAILED SELECT gen_dictionary('us_cities'); +--connection default +REVOKE SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries FROM 'mysql.session'@'localhost'; DROP TABLE mysql.masking_dictionaries; - +--echo # --echo # checks for an unprivileged user -CREATE USER udftest; ---connect(con1,localhost,udftest,,) ---connection con1 +--connection con_unpriv --error ER_CANT_INITIALIZE_UDF SELECT masking_dictionary_term_add('single_dict', 'entry'); @@ -101,15 +138,36 @@ SELECT masking_dictionary_term_remove('single_dict', 'entry'); --error ER_CANT_INITIALIZE_UDF SELECT masking_dictionary_remove('single_dict'); +--echo # +--echo # checking the case when 'mysql.session'@'localhost' has insufficient privileges --connection default ---echo # checking the case when mysql.masking_dictionaries is empty CREATE TABLE mysql.masking_dictionaries( Dictionary VARCHAR(256) NOT NULL, Term VARCHAR(256) NOT NULL, UNIQUE INDEX dictionary_term_idx (Dictionary, Term) ) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4; ---connection con1 +--error ER_COMMAND_SERVICE_BACKEND_FAILED +SELECT gen_dictionary('us_cities'); + +--connection con_priv +--error ER_COMMAND_SERVICE_BACKEND_FAILED +SELECT masking_dictionaries_flush(); + +--connection default +GRANT SELECT ON mysql.masking_dictionaries TO 'mysql.session'@'localhost'; +--connection con_priv + +--error ER_COMMAND_SERVICE_BACKEND_FAILED +SELECT masking_dictionary_term_add('single_dict', 'entry'); + + +--echo # +--echo # checking the case when mysql.masking_dictionaries is empty +--connection default +GRANT INSERT, UPDATE, DELETE ON mysql.masking_dictionaries TO 'mysql.session'@'localhost'; +--connection con_unpriv + --let $assert_cond = gen_dictionary("us_cities") IS NULL --let $assert_text = gen_dictionary on an empty table must return NULL --source include/assert.inc @@ -126,11 +184,71 @@ CREATE TABLE mysql.masking_dictionaries( INSERT INTO mysql.masking_dictionaries VALUES('us_cities', 'city1'); INSERT INTO mysql.masking_dictionaries VALUES('us_cities', 'city2'); INSERT INTO mysql.masking_dictionaries VALUES('us_cities', 'city3'); -INSERT INTO mysql.masking_dictionaries VALUES('us_cities', 'city4'); INSERT INTO mysql.masking_dictionaries VALUES('укр_міста', 'місто1'); + +--let $assert_cond = gen_dictionary("us_cities") IS NULL +--let $assert_text = gen_dictionary on a existing but not flushed dictionary must return NULL +--source include/assert.inc + +--connection con_priv +SELECT masking_dictionaries_flush(); +--connection con_unpriv + +--let $assert_cond = [ SELECT GROUP_CONCAT(val ORDER BY val) = "city1,city2,city3" FROM (SELECT gen_dictionary("us_cities") AS term FROM SEQUENCE_TABLE(100) AS tt GROUP BY term) AS tbl(val) ] = 1 +--let $assert_text = the number of distinct US city names after the first insert and flush must be 3 +--source include/assert.inc + +--connection default +INSERT INTO mysql.masking_dictionaries VALUES('us_cities', 'city4'); INSERT INTO mysql.masking_dictionaries VALUES('укр_міста', 'місто2'); ---connection con1 +--connection con_unpriv +--let $assert_cond = [ SELECT GROUP_CONCAT(val ORDER BY val) = "city1,city2,city3" FROM (SELECT gen_dictionary("us_cities") AS term FROM SEQUENCE_TABLE(100) AS tt GROUP BY term) AS tbl(val) ] = 1 +--let $assert_text = the number of distinct US city names after the second insert but before flush must be 3 +--source include/assert.inc + +--connection con_priv +SELECT masking_dictionaries_flush(); +--connection con_unpriv + +--let $assert_cond = [ SELECT GROUP_CONCAT(val ORDER BY val) = "city1,city2,city3,city4" FROM (SELECT gen_dictionary("us_cities") AS term FROM SEQUENCE_TABLE(100) AS tt GROUP BY term) AS tbl(val) ] = 1 +--let $assert_text = the number of distinct US city names after the second insert and flush must be 4 +--source include/assert.inc + +--connection default +RENAME TABLE mysql.masking_dictionaries TO mysql.masking_dictionaries_hidden; +--connection con_unpriv + +--let $assert_cond = [ SELECT GROUP_CONCAT(val ORDER BY val) = "city1,city2,city3,city4" FROM (SELECT gen_dictionary("us_cities") AS term FROM SEQUENCE_TABLE(100) AS tt GROUP BY term) AS tbl(val) ] = 1 +--let $assert_text = the number of distinct US city names after hiding dict table must be 4 +--source include/assert.inc + +--connection con_priv +--error ER_COMMAND_SERVICE_BACKEND_FAILED +SELECT masking_dictionaries_flush(); +--connection con_unpriv + +--let $assert_cond = [ SELECT GROUP_CONCAT(val ORDER BY val) = "city1,city2,city3,city4" FROM (SELECT gen_dictionary("us_cities") AS term FROM SEQUENCE_TABLE(100) AS tt GROUP BY term) AS tbl(val) ] = 1 +--let $assert_text = the number of distinct US city names after dict unsuccessful flush must be 4 +--source include/assert.inc + +--connection default +RENAME TABLE mysql.masking_dictionaries_hidden TO mysql.masking_dictionaries; +--connection con_unpriv + +--let $assert_cond = [ SELECT GROUP_CONCAT(val ORDER BY val) = "city1,city2,city3,city4" FROM (SELECT gen_dictionary("us_cities") AS term FROM SEQUENCE_TABLE(100) AS tt GROUP BY term) AS tbl(val) ] = 1 +--let $assert_text = the number of distinct US city names after restoring dict table must be 4 +--source include/assert.inc + +--connection con_priv +SELECT masking_dictionaries_flush(); +--connection con_unpriv + +--let $assert_cond = [ SELECT GROUP_CONCAT(val ORDER BY val) = "city1,city2,city3,city4" FROM (SELECT gen_dictionary("us_cities") AS term FROM SEQUENCE_TABLE(100) AS tt GROUP BY term) AS tbl(val) ] = 1 +--let $assert_text = the number of distinct US city names after restoring dict table and flush must be 4 +--source include/assert.inc + + --let $assert_cond = gen_dictionary("de_cities") IS NULL --let $assert_text = gen_dictionary on a non-existing dictionary must return NULL --source include/assert.inc @@ -182,15 +300,11 @@ INSERT INTO mysql.masking_dictionaries VALUES('укр_міста', 'місто2' --source include/assert.inc --connection default ---disconnect con1 - DELETE FROM mysql.masking_dictionaries; +--connection con_priv +SELECT masking_dictionaries_flush(); -GRANT MASKING_DICTIONARIES_ADMIN ON *.* TO udftest; - ---connect(con1,localhost,udftest,,) ---connection con1 - +--echo # --echo # argument nullness checks for functions requiring MASKING_DICTIONARIES_ADMIN --error ER_UDF_ERROR SELECT masking_dictionary_term_add(NULL, 'entry'); @@ -206,6 +320,7 @@ SELECT masking_dictionary_term_remove('single_dict', NULL); SELECT masking_dictionary_remove(NULL); +--echo # --echo # checks for a user with MASKING_DICTIONARIES_ADMIN privilege using various character sets / collations --let $dollar = `SELECT _utf8mb4 0x24` @@ -228,40 +343,40 @@ while($i < $number_of_charsets) { --echo masking_dictionary_term_add checks --let $dictionary = single --let $term = term00_$term_suffix - --let $expression = masking_dictionary_term_add('$dictionary', CONVERT('$term' USING $current_charset)) - --let $regexp = - --let $value = 1 - --let $expected_charset = utf8mb4 - --source check_expression.inc + --let $assert_cond = masking_dictionary_term_add("$dictionary", CONVERT("$term" USING $current_charset)) = 1 + --let $assert_text = '$assert_cond' must succeed + --source include/assert.inc - --let $assert_cond = masking_dictionary_term_add("$dictionary", CONVERT("$term" USING $current_charset)) IS NULL + --let $assert_cond = masking_dictionary_term_add("$dictionary", CONVERT("$term" USING $current_charset)) = 0 --let $assert_text = second masking_dictionary_term_add must fail --source include/assert.inc --let $dictionary = dictionary --let $term = term10_$term_suffix - --let $expression = masking_dictionary_term_add('$dictionary', CONVERT('$term' USING $current_charset)) - --let $regexp = - --let $value = 1 - --let $expected_charset = utf8mb4 - --source check_expression.inc + --let $assert_cond = masking_dictionary_term_add("$dictionary", CONVERT("$term" USING $current_charset)) = 1 + --let $assert_text = '$assert_cond' must succeed + --source include/assert.inc --let $term = term11_$term_suffix - --let $expression = masking_dictionary_term_add('$dictionary', CONVERT('$term' USING $current_charset)) - --source check_expression.inc + --let $assert_cond = masking_dictionary_term_add("$dictionary", CONVERT("$term" USING $current_charset)) = 1 + --let $assert_text = '$assert_cond' must succeed + --source include/assert.inc --let $dictionary = словник --let $term = term20_$term_suffix - --let $expression = masking_dictionary_term_add('$dictionary', CONVERT('$term' USING $current_charset)) - --source check_expression.inc + --let $assert_cond = masking_dictionary_term_add("$dictionary", CONVERT("$term" USING $current_charset)) = 1 + --let $assert_text = '$assert_cond' must succeed + --source include/assert.inc --let $term = term21_$term_suffix - --let $expression = masking_dictionary_term_add('$dictionary', CONVERT('$term' USING $current_charset)) - --source check_expression.inc + --let $assert_cond = masking_dictionary_term_add("$dictionary", CONVERT("$term" USING $current_charset)) = 1 + --let $assert_text = '$assert_cond' must succeed + --source include/assert.inc --let $term = term22_$term_suffix - --let $expression = masking_dictionary_term_add('$dictionary', CONVERT('$term' USING $current_charset)) - --source check_expression.inc + --let $assert_cond = masking_dictionary_term_add("$dictionary", CONVERT("$term" USING $current_charset)) = 1 + --let $assert_text = '$assert_cond' must succeed + --source include/assert.inc --connection default --let $assert_cond = [ SELECT COUNT(*) FROM mysql.masking_dictionaries ] = 6 @@ -269,7 +384,7 @@ while($i < $number_of_charsets) { --source include/assert.inc - --connection con1 + --connection con_priv --let $expected_charset = utf8mb4 --let $iteration = 0 while ($iteration < $number_of_iterations) { @@ -322,13 +437,11 @@ while($i < $number_of_charsets) { --echo masking_dictionary_term_remove checks --let $dictionary = single --let $term = term00_$term_suffix - --let $expression = masking_dictionary_term_remove('$dictionary', CONVERT('$term' USING $current_charset)) - --let $regexp = - --let $value = 1 - --let $expected_charset = utf8mb4 - --source check_expression.inc + --let $assert_cond = masking_dictionary_term_remove("$dictionary", CONVERT("$term" USING $current_charset)) = 1 + --let $assert_text = '$assert_cond' must succeed + --source include/assert.inc - --let $assert_cond = masking_dictionary_term_remove("$dictionary", CONVERT("$term" USING $current_charset)) IS NULL + --let $assert_cond = masking_dictionary_term_remove("$dictionary", CONVERT("$term" USING $current_charset)) = 0 --let $assert_text = second masking_dictionary_term_remove must fail --source include/assert.inc @@ -338,16 +451,14 @@ while($i < $number_of_charsets) { --source include/assert.inc - --connection con1 + --connection con_priv --echo masking_dictionary_remove checks --let $dictionary = dictionary - --let $expression = masking_dictionary_remove('$dictionary') - --let $regexp = - --let $value = 1 - --let $expected_charset = utf8mb4 - --source check_expression.inc + --let $assert_cond = masking_dictionary_remove("$dictionary") = 1 + --let $assert_text = '$assert_cond' must succeed + --source include/assert.inc - --let $assert_cond = masking_dictionary_remove("$dictionary") IS NULL + --let $assert_cond = masking_dictionary_remove("$dictionary") =0 --let $assert_text = second masking_dictionary_remove must fail --source include/assert.inc @@ -356,32 +467,33 @@ while($i < $number_of_charsets) { --let $assert_text = mysql.masking_dictionaries must have 3 records --source include/assert.inc - --connection con1 + --connection con_priv --let $dictionary = словник - --let $expression = masking_dictionary_remove('$dictionary') - --let $regexp = - --let $value = 1 - --let $expected_charset = utf8mb4 - --source check_expression.inc + --let $assert_cond = masking_dictionary_remove("$dictionary") = 1 + --let $assert_text = '$assert_cond' must succeed + --source include/assert.inc --connection default --let $assert_cond = [ SELECT COUNT(*) FROM mysql.masking_dictionaries ] = 0 --let $assert_text = mysql.masking_dictionaries must have 0 records --source include/assert.inc - --connection con1 + --connection con_priv --echo ################################ --inc $i } --connection default ---disconnect con1 +--disconnect con_priv +--disconnect con_unpriv -DROP USER udftest; +DROP USER udftest_unpriv@localhost; +DROP USER udftest_priv@localhost; UNINSTALL COMPONENT 'file://component_masking_functions'; +REVOKE SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries FROM 'mysql.session'@'localhost'; DROP TABLE mysql.masking_dictionaries; --source include/wait_until_count_sessions.inc diff --git a/mysql-test/suite/component_masking_functions/t/flusher_thread_connection_reuse-master.opt b/mysql-test/suite/component_masking_functions/t/flusher_thread_connection_reuse-master.opt new file mode 100644 index 000000000000..2775a1988fee --- /dev/null +++ b/mysql-test/suite/component_masking_functions/t/flusher_thread_connection_reuse-master.opt @@ -0,0 +1 @@ +--loose-masking_functions.dictionaries_flush_interval_seconds=1 diff --git a/mysql-test/suite/component_masking_functions/t/flusher_thread_connection_reuse.test b/mysql-test/suite/component_masking_functions/t/flusher_thread_connection_reuse.test new file mode 100644 index 000000000000..9142406dd6fb --- /dev/null +++ b/mysql-test/suite/component_masking_functions/t/flusher_thread_connection_reuse.test @@ -0,0 +1,42 @@ +--source include/have_debug.inc +--source include/have_debug_sync.inc +--source include/have_masking_functions_component.inc + +--source include/count_sessions.inc + +CREATE TABLE mysql.masking_dictionaries( + Dictionary VARCHAR(256) NOT NULL, + Term VARCHAR(256) NOT NULL, + UNIQUE INDEX dictionary_term_idx (Dictionary, Term) +) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4; +GRANT SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries TO 'mysql.session'@'localhost'; +INSERT INTO mysql.masking_dictionaries VALUES('dict1', 'term1'); + + +SET GLOBAL debug = '+d,enable_masking_functions_flusher_create_sync,enable_masking_functions_flush_thread_sync,enable_masking_functions_flush_thread_double_pass'; +SET debug_sync = 'masking_functions_after_flusher_create WAIT_FOR masking_functions_after_flusher_create_signal'; + +send INSTALL COMPONENT 'file://component_masking_functions'; + +--connect(con_priv,localhost,root,,) +SET debug_sync = 'now SIGNAL masking_functions_before_cache_reload_signal WAIT_FOR masking_functions_after_cache_reload_signal'; + +SET debug_sync = 'now SIGNAL masking_functions_after_flusher_create_signal'; + +--connection default +--reap + +--disconnect con_priv + +SET debug_sync = 'now SIGNAL masking_functions_before_cache_reload_signal WAIT_FOR masking_functions_after_cache_reload_signal'; + +UNINSTALL COMPONENT 'file://component_masking_functions'; + +SET GLOBAL debug = '-d,enable_masking_functions_flusher_create_sync,enable_masking_functions_flush_thread_sync,enable_masking_functions_flush_thread_double_pass'; + +REVOKE SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries FROM 'mysql.session'@'localhost'; +DROP TABLE mysql.masking_dictionaries; + +--source include/wait_until_count_sessions.inc + +SET debug_sync = 'RESET'; diff --git a/mysql-test/suite/component_masking_functions/t/flusher_thread_immediate_restart-master.opt b/mysql-test/suite/component_masking_functions/t/flusher_thread_immediate_restart-master.opt new file mode 100644 index 000000000000..2775a1988fee --- /dev/null +++ b/mysql-test/suite/component_masking_functions/t/flusher_thread_immediate_restart-master.opt @@ -0,0 +1 @@ +--loose-masking_functions.dictionaries_flush_interval_seconds=1 diff --git a/mysql-test/suite/component_masking_functions/t/flusher_thread_immediate_restart.test b/mysql-test/suite/component_masking_functions/t/flusher_thread_immediate_restart.test new file mode 100644 index 000000000000..3f0cae7fdb5e --- /dev/null +++ b/mysql-test/suite/component_masking_functions/t/flusher_thread_immediate_restart.test @@ -0,0 +1,23 @@ +--source include/have_masking_functions_component.inc + +CALL mtr.add_suppression("Flusher thread terminated while waiting for session server"); +CALL mtr.add_suppression("Flusher thread terminated after creating internal connection"); +CALL mtr.add_suppression("Server shutdown requested while attempting to establish flusher thread connection"); + +CREATE TABLE mysql.masking_dictionaries( + Dictionary VARCHAR(256) NOT NULL, + Term VARCHAR(256) NOT NULL, + UNIQUE INDEX dictionary_term_idx (Dictionary, Term) +) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4; +GRANT SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries TO 'mysql.session'@'localhost'; +INSERT INTO mysql.masking_dictionaries VALUES('dict1', 'term1'); + + +INSTALL COMPONENT 'file://component_masking_functions'; + +--source include/restart_mysqld.inc + +UNINSTALL COMPONENT 'file://component_masking_functions'; + +REVOKE SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries FROM 'mysql.session'@'localhost'; +DROP TABLE mysql.masking_dictionaries; diff --git a/mysql-test/suite/component_masking_functions/t/flusher_thread_suspend_resume-master.opt b/mysql-test/suite/component_masking_functions/t/flusher_thread_suspend_resume-master.opt new file mode 100644 index 000000000000..2775a1988fee --- /dev/null +++ b/mysql-test/suite/component_masking_functions/t/flusher_thread_suspend_resume-master.opt @@ -0,0 +1 @@ +--loose-masking_functions.dictionaries_flush_interval_seconds=1 diff --git a/mysql-test/suite/component_masking_functions/t/flusher_thread_suspend_resume.test b/mysql-test/suite/component_masking_functions/t/flusher_thread_suspend_resume.test new file mode 100644 index 000000000000..39ce2b92679f --- /dev/null +++ b/mysql-test/suite/component_masking_functions/t/flusher_thread_suspend_resume.test @@ -0,0 +1,71 @@ +--source include/have_debug.inc +--source include/have_debug_sync.inc +--source include/have_masking_functions_component.inc + +# as this test can be run with --repeat=N, we need to be able to select log error records +# only from the current run - to do so we identify the timestamp of the oldest record +# at the beginning of the execution +--let $startup_ts = `SELECT MAX(logged) FROM performance_schema.error_log` + +--let $expected_flusher_thread_error = Exception during reloading dictionary cache +eval CALL mtr.add_suppression("$expected_flusher_thread_error"); + +--source include/count_sessions.inc + +SET GLOBAL debug = '+d,enable_masking_functions_flusher_create_sync,enable_masking_functions_flush_thread_sync,enable_masking_functions_flush_thread_double_pass'; +SET debug_sync = 'masking_functions_after_flusher_create WAIT_FOR masking_functions_after_flusher_create_signal'; + +send INSTALL COMPONENT 'file://component_masking_functions'; + +--connect(con_priv,localhost,root,,) +SET debug_sync = 'now SIGNAL masking_functions_before_cache_reload_signal WAIT_FOR masking_functions_after_cache_reload_signal'; + +--let $assert_cond = [SELECT COUNT(*) FROM performance_schema.error_log WHERE logged >= "$startup_ts" AND prio = "Error" AND data LIKE "%$expected_flusher_thread_error%"] = 1 +--let $assert_text = Server error log must contain an error record from the Dictionary Flusher thread +--source include/assert.inc + +SET debug_sync = 'now SIGNAL masking_functions_after_flusher_create_signal'; + +--connection default +--reap + +--disconnect con_priv + +CREATE TABLE mysql.masking_dictionaries( + Dictionary VARCHAR(256) NOT NULL, + Term VARCHAR(256) NOT NULL, + UNIQUE INDEX dictionary_term_idx (Dictionary, Term) +) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4; +GRANT SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries TO 'mysql.session'@'localhost'; + +# Even though the flusher thread is suspended, here the very first call to the 'gen_dictionary()' +# will trigger cache reloading +INSERT INTO mysql.masking_dictionaries VALUES('dict1', 'term1'); +--let $assert_cond = gen_dictionary("dict1") = "term1" +--let $assert_text = The very first call to gen_dictionary("dict1") should trigger inplace cache reload and return "term1" +--source include/assert.inc + +INSERT INTO mysql.masking_dictionaries VALUES('dict2', 'term2'); +--let $assert_cond = gen_dictionary("dict2") IS NULL +--let $assert_text = The very first call to gen_dictionary("dict2") should not trigger inplace cache reload and return NULL +--source include/assert.inc + +SET debug_sync = 'now SIGNAL masking_functions_before_cache_reload_signal WAIT_FOR masking_functions_after_cache_reload_signal'; +--let $assert_cond = gen_dictionary("dict1") = "term1" +--let $assert_text = After resuming flusher thread gen_dictionary("dict1") should still return "term1" +--source include/assert.inc + +--let $assert_cond = gen_dictionary("dict2") = "term2" +--let $assert_text = After resuming flusher thread gen_dictionary("dict2") should get new values from the cache updated by the flusher and return "term2" +--source include/assert.inc + +UNINSTALL COMPONENT 'file://component_masking_functions'; + +SET GLOBAL debug = '-d,enable_masking_functions_flusher_create_sync,enable_masking_functions_flush_thread_sync,enable_masking_functions_flush_thread_double_pass'; + +REVOKE SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries FROM 'mysql.session'@'localhost'; +DROP TABLE mysql.masking_dictionaries; + +--source include/wait_until_count_sessions.inc + +SET debug_sync = 'RESET'; diff --git a/mysql-test/suite/component_masking_functions/t/global_autocommit_off.test b/mysql-test/suite/component_masking_functions/t/global_autocommit_off.test index 7c4a9231950a..ea84322de60b 100644 --- a/mysql-test/suite/component_masking_functions/t/global_autocommit_off.test +++ b/mysql-test/suite/component_masking_functions/t/global_autocommit_off.test @@ -13,6 +13,7 @@ CREATE TABLE mysql.masking_dictionaries( Term VARCHAR(256) NOT NULL, UNIQUE INDEX dictionary_term_idx (Dictionary, Term) ) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4; +GRANT SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries TO 'mysql.session'@'localhost'; --let $current_user = `SELECT USER()` eval GRANT MASKING_DICTIONARIES_ADMIN ON *.* TO $current_user; @@ -34,6 +35,7 @@ SET @@global.autocommit = OFF; --source include/assert.inc eval REVOKE MASKING_DICTIONARIES_ADMIN ON *.* FROM $current_user; +REVOKE SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries FROM 'mysql.session'@'localhost'; DROP TABLE mysql.masking_dictionaries; diff --git a/mysql-test/suite/component_masking_functions/t/rpl_dictionaries_flush_interval-master.opt b/mysql-test/suite/component_masking_functions/t/rpl_dictionaries_flush_interval-master.opt new file mode 100644 index 000000000000..2775a1988fee --- /dev/null +++ b/mysql-test/suite/component_masking_functions/t/rpl_dictionaries_flush_interval-master.opt @@ -0,0 +1 @@ +--loose-masking_functions.dictionaries_flush_interval_seconds=1 diff --git a/mysql-test/suite/component_masking_functions/t/rpl_dictionaries_flush_interval-slave.opt b/mysql-test/suite/component_masking_functions/t/rpl_dictionaries_flush_interval-slave.opt new file mode 100644 index 000000000000..2775a1988fee --- /dev/null +++ b/mysql-test/suite/component_masking_functions/t/rpl_dictionaries_flush_interval-slave.opt @@ -0,0 +1 @@ +--loose-masking_functions.dictionaries_flush_interval_seconds=1 diff --git a/mysql-test/suite/component_masking_functions/t/rpl_dictionaries_flush_interval.test b/mysql-test/suite/component_masking_functions/t/rpl_dictionaries_flush_interval.test new file mode 100644 index 000000000000..3dba4f244da2 --- /dev/null +++ b/mysql-test/suite/component_masking_functions/t/rpl_dictionaries_flush_interval.test @@ -0,0 +1,140 @@ +--source include/have_debug.inc +--source include/have_debug_sync.inc +--source include/have_masking_functions_component.inc + +--source include/master-slave.inc + +--source include/rpl_connection_master.inc +CREATE TABLE mysql.masking_dictionaries( + Dictionary VARCHAR(256) NOT NULL, + Term VARCHAR(256) NOT NULL, + UNIQUE INDEX dictionary_term_idx (Dictionary, Term) +) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4; +SET sql_log_bin = OFF; +GRANT SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries TO 'mysql.session'@'localhost'; +SET sql_log_bin = ON; + + +--source include/rpl_sync.inc +--source include/rpl_connection_slave.inc +GRANT SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries TO 'mysql.session'@'localhost'; + +--source include/rpl_connection_master.inc +SET GLOBAL debug ='+d,enable_masking_functions_flush_thread_sync'; +INSTALL COMPONENT 'file://component_masking_functions'; +SET sql_log_bin = OFF; +GRANT MASKING_DICTIONARIES_ADMIN ON *.* TO root@localhost; +SET sql_log_bin = ON; + + +--source include/rpl_connection_slave.inc +SET GLOBAL debug ='+d,enable_masking_functions_flush_thread_sync'; +INSTALL COMPONENT 'file://component_masking_functions'; +GRANT MASKING_DICTIONARIES_ADMIN ON *.* TO root@localhost; + + +--source include/rpl_connection_master.inc +--let $assert_cond = gen_dictionary("dict1") IS NULL +--let $assert_text = The very first call to gen_dictionary("dict1") on source must load empty cache and return NULL +--source include/assert.inc + + +--source include/rpl_sync.inc +--source include/rpl_connection_slave.inc +--let $assert_cond = gen_dictionary("dict1") IS NULL +--let $assert_text = The very first call to gen_dictionary("dict1") on replica must load empty cache and return NULL +--source include/assert.inc + + +--source include/rpl_connection_master.inc +SELECT masking_dictionary_term_add('dict1', 'term1'); +SELECT masking_dictionary_term_add('dict2', 'term2'); + +--let $assert_cond = gen_dictionary("dict1") = "term1" +--let $assert_text = The call to gen_dictionary("dict1") after inserting terms on source must return "term1" +--source include/assert.inc + +--let $assert_cond = gen_dictionary("dict2") = "term2" +--let $assert_text = The call to gen_dictionary("dict2") after inserting terms on source must return "term2" +--source include/assert.inc + + +--source include/rpl_sync.inc +--source include/rpl_connection_slave.inc +--let $assert_cond = [SELECT COUNT(*) FROM mysql.masking_dictionaries] = 2 +--let $assert_text = The number of dictionary terms on replica after synchronization must be 2 +--source include/assert.inc + +--let $assert_cond = gen_dictionary("dict1") IS NULL +--let $assert_text = The call to gen_dictionary("dict1") on replica after synchronization but without reloading cache must return NULL +--source include/assert.inc + +--let $assert_cond = gen_dictionary("dict2") IS NULL +--let $assert_text = The call to gen_dictionary("dict1") on replica after synchronization but without reloading cache must return NULL +--source include/assert.inc + +SELECT masking_dictionaries_flush(); + +--let $assert_cond = gen_dictionary("dict1") = "term1" +--let $assert_text = The call to gen_dictionary("dict1") on replica after synchronization and reloading cache must return "term1" +--source include/assert.inc + +--let $assert_cond = gen_dictionary("dict2") = "term2" +--let $assert_text = The call to gen_dictionary("dict1") on replica after synchronization and reloading cache must return "term2" +--source include/assert.inc + + +--source include/rpl_connection_master.inc +INSERT INTO mysql.masking_dictionaries VALUES ('dict3', 'term3'); + +--let $assert_cond = gen_dictionary("dict3") IS NULL +--let $assert_text = The call to gen_dictionary("dict3") on source with suspended flusher thread must return NULL +--source include/assert.inc + +SET debug_sync = 'now SIGNAL masking_functions_before_cache_reload_signal WAIT_FOR masking_functions_after_cache_reload_signal'; + +--let $assert_cond = gen_dictionary("dict3") = "term3" +--let $assert_text = The call to gen_dictionary("dict3") on source after resuming flusher thread must return "term3" +--source include/assert.inc + + +--source include/rpl_sync.inc +--source include/rpl_connection_slave.inc +--let $assert_cond = [SELECT COUNT(*) FROM mysql.masking_dictionaries] = 3 +--let $assert_text = The number of dictionary terms on replica after the second synchronization must be 3 +--source include/assert.inc + +--let $assert_cond = gen_dictionary("dict3") IS NULL +--let $assert_text = The call to gen_dictionary("dict3") on replica with suspended flusher thread must return NULL +--source include/assert.inc + +SET debug_sync = 'now SIGNAL masking_functions_before_cache_reload_signal WAIT_FOR masking_functions_after_cache_reload_signal'; + +--let $assert_cond = gen_dictionary("dict3") = "term3" +--let $assert_text = The call to gen_dictionary("dict3") on replica after resuming flusher thread must return "term3" +--source include/assert.inc + +# +# Cleanup +--source include/rpl_connection_slave.inc +REVOKE MASKING_DICTIONARIES_ADMIN ON *.* FROM root@localhost; +SET GLOBAL debug ='-d,enable_masking_functions_flush_thread_sync'; +UNINSTALL COMPONENT 'file://component_masking_functions'; +REVOKE SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries FROM 'mysql.session'@'localhost'; +SET debug_sync = 'RESET'; + +--source include/rpl_connection_master.inc +SET sql_log_bin = OFF; +REVOKE MASKING_DICTIONARIES_ADMIN ON *.* FROM root@localhost; +SET sql_log_bin = ON; +SET GLOBAL debug ='-d,enable_masking_functions_flush_thread_sync'; +UNINSTALL COMPONENT 'file://component_masking_functions'; +SET sql_log_bin = OFF; +REVOKE SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries FROM 'mysql.session'@'localhost'; +SET sql_log_bin = ON; +SET debug_sync = 'RESET'; + +DROP TABLE mysql.masking_dictionaries; + +--source include/rpl_sync.inc +--source include/rpl_end.inc diff --git a/mysql-test/suite/component_masking_functions/t/rpl_function_determinism.inc b/mysql-test/suite/component_masking_functions/t/rpl_function_determinism.inc index 7e0bebdd6541..d93ae0dcb8fa 100644 --- a/mysql-test/suite/component_masking_functions/t/rpl_function_determinism.inc +++ b/mysql-test/suite/component_masking_functions/t/rpl_function_determinism.inc @@ -137,6 +137,7 @@ CREATE TABLE mysql.masking_dictionaries( Term VARCHAR(256) NOT NULL, UNIQUE INDEX dictionary_term_idx (Dictionary, Term) ) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4; +GRANT SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries TO 'mysql.session'@'localhost'; --let $current_user = `SELECT USER()` eval GRANT MASKING_DICTIONARIES_ADMIN ON *.* TO $current_user; @@ -182,6 +183,7 @@ SELECT masking_dictionary_remove('dict1'); --source include/rpl_connection_master.inc eval REVOKE MASKING_DICTIONARIES_ADMIN ON *.* FROM $current_user; +REVOKE SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries FROM 'mysql.session'@'localhost'; DROP TABLE mysql.masking_dictionaries; --source include/rpl_connection_slave.inc diff --git a/mysql-test/suite/component_masking_functions/t/sys_var_dictionaries_flush_interval_seconds_basic.test b/mysql-test/suite/component_masking_functions/t/sys_var_dictionaries_flush_interval_seconds_basic.test new file mode 100644 index 000000000000..7a260a4c4549 --- /dev/null +++ b/mysql-test/suite/component_masking_functions/t/sys_var_dictionaries_flush_interval_seconds_basic.test @@ -0,0 +1,55 @@ +--source include/have_masking_functions_component.inc + +CALL mtr.add_suppression("Flusher thread terminated while waiting for session server"); +CALL mtr.add_suppression("Flusher thread terminated after creating internal connection"); +CALL mtr.add_suppression("Server shutdown requested while attempting to establish flusher thread connection"); + +INSTALL COMPONENT 'file://component_masking_functions'; + +# No running flusher thread with default settings +SELECT @@global.masking_functions.dictionaries_flush_interval_seconds; +SELECT NAME FROM performance_schema.threads WHERE NAME LIKE "%masking_functions%"; + +--error ER_INCORRECT_GLOBAL_LOCAL_VAR +SET GLOBAL masking_functions.dictionaries_flush_interval_seconds=100; + +--error ER_INCORRECT_GLOBAL_LOCAL_VAR +SET SESSION masking_functions.dictionaries_flush_interval_seconds=100; + +# Make sure dict flusher process is running +CREATE TABLE mysql.masking_dictionaries( + Dictionary VARCHAR(256) NOT NULL, + Term VARCHAR(256) NOT NULL, + UNIQUE INDEX dictionary_term_idx (Dictionary, Term) +) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4; +GRANT SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries TO 'mysql.session'@'localhost'; + +--let $restart_parameters="restart: --masking_functions.dictionaries_flush_interval_seconds=100" +--source include/restart_mysqld.inc + +SELECT @@global.masking_functions.dictionaries_flush_interval_seconds; + +CREATE USER udftest_priv@localhost; +GRANT MASKING_DICTIONARIES_ADMIN ON *.* TO udftest_priv@localhost; +--connect(con_priv,localhost,udftest_priv,,) + +SELECT masking_dictionary_term_add('single_dict_1', 'entry_1'); + +# Flusher thread is active +--connection default +SELECT NAME FROM performance_schema.threads WHERE NAME LIKE "%masking_functions%"; + +# +# Cleanup +--disconnect con_priv + +REVOKE MASKING_DICTIONARIES_ADMIN ON *.* FROM udftest_priv@localhost; +UNINSTALL COMPONENT 'file://component_masking_functions'; + +REVOKE SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries FROM 'mysql.session'@'localhost'; + +DROP USER udftest_priv@localhost; +DROP TABLE mysql.masking_dictionaries; + +--let $restart_parameters="restart:" +--source include/restart_mysqld.inc diff --git a/mysql-test/suite/component_masking_functions/t/sys_var_masking_database.test b/mysql-test/suite/component_masking_functions/t/sys_var_masking_database.test new file mode 100644 index 000000000000..7157fa847a56 --- /dev/null +++ b/mysql-test/suite/component_masking_functions/t/sys_var_masking_database.test @@ -0,0 +1,54 @@ +--source include/have_masking_functions_component.inc +--source include/count_sessions.inc + +INSTALL COMPONENT 'file://component_masking_functions'; + +--error ER_INCORRECT_GLOBAL_LOCAL_VAR +SET GLOBAL masking_functions.masking_database=dict_db; + +--error ER_INCORRECT_GLOBAL_LOCAL_VAR +SET SESSION masking_functions.masking_database=dict_db; + +# Empty DB name +--let $error_log_file = $MYSQLTEST_VARDIR/tmp/masking_functions_error.err +--let $do_not_echo_parameters = 1 +--let $restart_parameters="restart: --log-error=$error_log_file --masking-functions.masking-database=''" +--source include/restart_mysqld.inc + +--let $assert_text = Bad masking_functions.masking_database value +--let $assert_file = $error_log_file +--let $assert_select = Bad masking_functions.masking_database value +--let $assert_count = 1 +--source include/assert_grep.inc + +CREATE DATABASE dict_db; +CREATE TABLE dict_db.masking_dictionaries( + Dictionary VARCHAR(256) NOT NULL, + Term VARCHAR(256) NOT NULL, + UNIQUE INDEX dictionary_term_idx (Dictionary, Term) +) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4; +GRANT SELECT, INSERT, UPDATE, DELETE ON dict_db.masking_dictionaries TO 'mysql.session'@'localhost'; + +--let $restart_parameters="restart: --log-error=$error_log_file --masking-functions.masking-database='dict_db'" +--source include/restart_mysqld.inc + +CREATE USER udftest_priv@localhost; +GRANT MASKING_DICTIONARIES_ADMIN ON *.* TO udftest_priv@localhost; +--connect(con_priv,localhost,udftest_priv,,) + +SELECT masking_dictionary_term_add('single_dict', 'entry'); +SELECT gen_dictionary('single_dict'); + +--connection default +--disconnect con_priv + +# +# Cleanup +UNINSTALL COMPONENT 'file://component_masking_functions'; +REVOKE SELECT, INSERT, UPDATE, DELETE ON dict_db.masking_dictionaries FROM 'mysql.session'@'localhost'; +DROP TABLE dict_db.masking_dictionaries; +DROP DATABASE dict_db; +DROP USER udftest_priv@localhost; + +--remove_file $error_log_file +--source include/wait_until_count_sessions.inc From 5316d36db778214820e0558d30f2188c356ec292 Mon Sep 17 00:00:00 2001 From: Yura Sorokin Date: Tue, 21 Jan 2025 02:33:53 +0100 Subject: [PATCH 08/11] PS-9148 feature: 8.4-specific MTR fixes https://perconadev.atlassian.net/browse/PS-9148 Changed replication terminology inthe 'component_masking_functions.rpl_dictionaries_flush_interval' MTR test case. --- .../r/rpl_dictionaries_flush_interval.result | 14 +++---- .../t/rpl_dictionaries_flush_interval.test | 38 +++++++++---------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/mysql-test/suite/component_masking_functions/r/rpl_dictionaries_flush_interval.result b/mysql-test/suite/component_masking_functions/r/rpl_dictionaries_flush_interval.result index c311a0c32182..f97e8a2ce51e 100644 --- a/mysql-test/suite/component_masking_functions/r/rpl_dictionaries_flush_interval.result +++ b/mysql-test/suite/component_masking_functions/r/rpl_dictionaries_flush_interval.result @@ -1,4 +1,4 @@ -include/master-slave.inc +include/rpl/init_source_replica.inc Warnings: Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. Note #### Storing MySQL user name or password information in the connection metadata repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START REPLICA; see the 'START REPLICA Syntax' in the MySQL Manual for more information. @@ -12,7 +12,7 @@ UNIQUE INDEX dictionary_term_idx (Dictionary, Term) SET sql_log_bin = OFF; GRANT SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries TO 'mysql.session'@'localhost'; SET sql_log_bin = ON; -include/rpl_sync.inc +include/rpl/sync.inc [connection slave] GRANT SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries TO 'mysql.session'@'localhost'; [connection master] @@ -27,7 +27,7 @@ INSTALL COMPONENT 'file://component_masking_functions'; GRANT MASKING_DICTIONARIES_ADMIN ON *.* TO root@localhost; [connection master] include/assert.inc [The very first call to gen_dictionary("dict1") on source must load empty cache and return NULL] -include/rpl_sync.inc +include/rpl/sync.inc [connection slave] include/assert.inc [The very first call to gen_dictionary("dict1") on replica must load empty cache and return NULL] [connection master] @@ -39,7 +39,7 @@ masking_dictionary_term_add('dict2', 'term2') 1 include/assert.inc [The call to gen_dictionary("dict1") after inserting terms on source must return "term1"] include/assert.inc [The call to gen_dictionary("dict2") after inserting terms on source must return "term2"] -include/rpl_sync.inc +include/rpl/sync.inc [connection slave] include/assert.inc [The number of dictionary terms on replica after synchronization must be 2] include/assert.inc [The call to gen_dictionary("dict1") on replica after synchronization but without reloading cache must return NULL] @@ -54,7 +54,7 @@ INSERT INTO mysql.masking_dictionaries VALUES ('dict3', 'term3'); include/assert.inc [The call to gen_dictionary("dict3") on source with suspended flusher thread must return NULL] SET debug_sync = 'now SIGNAL masking_functions_before_cache_reload_signal WAIT_FOR masking_functions_after_cache_reload_signal'; include/assert.inc [The call to gen_dictionary("dict3") on source after resuming flusher thread must return "term3"] -include/rpl_sync.inc +include/rpl/sync.inc [connection slave] include/assert.inc [The number of dictionary terms on replica after the second synchronization must be 3] include/assert.inc [The call to gen_dictionary("dict3") on replica with suspended flusher thread must return NULL] @@ -77,5 +77,5 @@ REVOKE SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries FROM 'mysql. SET sql_log_bin = ON; SET debug_sync = 'RESET'; DROP TABLE mysql.masking_dictionaries; -include/rpl_sync.inc -include/rpl_end.inc +include/rpl/sync.inc +include/rpl/deinit.inc diff --git a/mysql-test/suite/component_masking_functions/t/rpl_dictionaries_flush_interval.test b/mysql-test/suite/component_masking_functions/t/rpl_dictionaries_flush_interval.test index 3dba4f244da2..46a36511ec29 100644 --- a/mysql-test/suite/component_masking_functions/t/rpl_dictionaries_flush_interval.test +++ b/mysql-test/suite/component_masking_functions/t/rpl_dictionaries_flush_interval.test @@ -2,9 +2,9 @@ --source include/have_debug_sync.inc --source include/have_masking_functions_component.inc ---source include/master-slave.inc +--source include/rpl/init_source_replica.inc ---source include/rpl_connection_master.inc +--source include/rpl/connection_source.inc CREATE TABLE mysql.masking_dictionaries( Dictionary VARCHAR(256) NOT NULL, Term VARCHAR(256) NOT NULL, @@ -15,11 +15,11 @@ GRANT SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries TO 'mysql.ses SET sql_log_bin = ON; ---source include/rpl_sync.inc ---source include/rpl_connection_slave.inc +--source include/rpl/sync.inc +--source include/rpl/connection_replica.inc GRANT SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries TO 'mysql.session'@'localhost'; ---source include/rpl_connection_master.inc +--source include/rpl/connection_source.inc SET GLOBAL debug ='+d,enable_masking_functions_flush_thread_sync'; INSTALL COMPONENT 'file://component_masking_functions'; SET sql_log_bin = OFF; @@ -27,26 +27,26 @@ GRANT MASKING_DICTIONARIES_ADMIN ON *.* TO root@localhost; SET sql_log_bin = ON; ---source include/rpl_connection_slave.inc +--source include/rpl/connection_replica.inc SET GLOBAL debug ='+d,enable_masking_functions_flush_thread_sync'; INSTALL COMPONENT 'file://component_masking_functions'; GRANT MASKING_DICTIONARIES_ADMIN ON *.* TO root@localhost; ---source include/rpl_connection_master.inc +--source include/rpl/connection_source.inc --let $assert_cond = gen_dictionary("dict1") IS NULL --let $assert_text = The very first call to gen_dictionary("dict1") on source must load empty cache and return NULL --source include/assert.inc ---source include/rpl_sync.inc ---source include/rpl_connection_slave.inc +--source include/rpl/sync.inc +--source include/rpl/connection_replica.inc --let $assert_cond = gen_dictionary("dict1") IS NULL --let $assert_text = The very first call to gen_dictionary("dict1") on replica must load empty cache and return NULL --source include/assert.inc ---source include/rpl_connection_master.inc +--source include/rpl/connection_source.inc SELECT masking_dictionary_term_add('dict1', 'term1'); SELECT masking_dictionary_term_add('dict2', 'term2'); @@ -59,8 +59,8 @@ SELECT masking_dictionary_term_add('dict2', 'term2'); --source include/assert.inc ---source include/rpl_sync.inc ---source include/rpl_connection_slave.inc +--source include/rpl/sync.inc +--source include/rpl/connection_replica.inc --let $assert_cond = [SELECT COUNT(*) FROM mysql.masking_dictionaries] = 2 --let $assert_text = The number of dictionary terms on replica after synchronization must be 2 --source include/assert.inc @@ -84,7 +84,7 @@ SELECT masking_dictionaries_flush(); --source include/assert.inc ---source include/rpl_connection_master.inc +--source include/rpl/connection_source.inc INSERT INTO mysql.masking_dictionaries VALUES ('dict3', 'term3'); --let $assert_cond = gen_dictionary("dict3") IS NULL @@ -98,8 +98,8 @@ SET debug_sync = 'now SIGNAL masking_functions_before_cache_reload_signal WAIT_F --source include/assert.inc ---source include/rpl_sync.inc ---source include/rpl_connection_slave.inc +--source include/rpl/sync.inc +--source include/rpl/connection_replica.inc --let $assert_cond = [SELECT COUNT(*) FROM mysql.masking_dictionaries] = 3 --let $assert_text = The number of dictionary terms on replica after the second synchronization must be 3 --source include/assert.inc @@ -116,14 +116,14 @@ SET debug_sync = 'now SIGNAL masking_functions_before_cache_reload_signal WAIT_F # # Cleanup ---source include/rpl_connection_slave.inc +--source include/rpl/connection_replica.inc REVOKE MASKING_DICTIONARIES_ADMIN ON *.* FROM root@localhost; SET GLOBAL debug ='-d,enable_masking_functions_flush_thread_sync'; UNINSTALL COMPONENT 'file://component_masking_functions'; REVOKE SELECT, INSERT, UPDATE, DELETE ON mysql.masking_dictionaries FROM 'mysql.session'@'localhost'; SET debug_sync = 'RESET'; ---source include/rpl_connection_master.inc +--source include/rpl/connection_source.inc SET sql_log_bin = OFF; REVOKE MASKING_DICTIONARIES_ADMIN ON *.* FROM root@localhost; SET sql_log_bin = ON; @@ -136,5 +136,5 @@ SET debug_sync = 'RESET'; DROP TABLE mysql.masking_dictionaries; ---source include/rpl_sync.inc ---source include/rpl_end.inc +--source include/rpl/sync.inc +--source include/rpl/deinit.inc From fa7b21662d2c7634b83ee8129cac859e4f7aa448 Mon Sep 17 00:00:00 2001 From: Yura Sorokin Date: Tue, 21 Jan 2025 03:48:09 +0100 Subject: [PATCH 09/11] PS-9148 feature: 8.4-specific c++20 improvements https://perconadev.atlassian.net/browse/PS-9148 Added minor c++20 improvements. --- .../masking_functions/src/component.cpp | 41 +++++++++---------- .../src/masking_functions/dictionary.cpp | 3 +- .../registration_routines.cpp | 2 - 3 files changed, 20 insertions(+), 26 deletions(-) diff --git a/components/masking_functions/src/component.cpp b/components/masking_functions/src/component.cpp index a5eb789a66c2..37a719f8e53f 100644 --- a/components/masking_functions/src/component.cpp +++ b/components/masking_functions/src/component.cpp @@ -125,34 +125,31 @@ mysql_service_status_t component_init() { masking_functions::primitive_singleton< masking_functions::string_service_tuple>::instance() = masking_functions::string_service_tuple{ - // TODO: convert this to designated initializers in c++20 - mysql_service_mysql_charset, - mysql_service_mysql_string_factory, - mysql_service_mysql_string_charset_converter, - mysql_service_mysql_string_get_data_in_charset, - mysql_service_mysql_string_append, - mysql_service_mysql_string_character_access, - mysql_service_mysql_string_byte_access, - mysql_service_mysql_string_reset, - mysql_service_mysql_string_substr, - mysql_service_mysql_string_compare}; + .charset = mysql_service_mysql_charset, + .factory = mysql_service_mysql_string_factory, + .converter = mysql_service_mysql_string_charset_converter, + .get_data_in_charset = mysql_service_mysql_string_get_data_in_charset, + .append = mysql_service_mysql_string_append, + .character_access = mysql_service_mysql_string_character_access, + .byte_access = mysql_service_mysql_string_byte_access, + .reset = mysql_service_mysql_string_reset, + .substr = mysql_service_mysql_string_substr, + .compare = mysql_service_mysql_string_compare}; masking_functions::primitive_singleton< masking_functions::command_service_tuple>::instance() = masking_functions::command_service_tuple{ - // TODO: convert this to designated initializers in c++20 - mysql_service_mysql_command_query, - mysql_service_mysql_command_query_result, - mysql_service_mysql_command_field_info, - mysql_service_mysql_command_options, - mysql_service_mysql_command_factory, - mysql_service_mysql_command_error_info, - mysql_service_mysql_command_thread}; + .query = mysql_service_mysql_command_query, + .query_result = mysql_service_mysql_command_query_result, + .field_info = mysql_service_mysql_command_field_info, + .options = mysql_service_mysql_command_options, + .factory = mysql_service_mysql_command_factory, + .error_info = mysql_service_mysql_command_error_info, + .thread = mysql_service_mysql_command_thread}; masking_functions::primitive_singleton< masking_functions::component_sys_variable_service_tuple>::instance() = masking_functions::component_sys_variable_service_tuple{ - // TODO: convert this to designated initializers in c++20 - mysql_service_component_sys_variable_register, - mysql_service_component_sys_variable_unregister}; + .registrator = mysql_service_component_sys_variable_register, + .unregistrator = mysql_service_component_sys_variable_unregister}; // here we use a custom error reporting function // 'masking_functions_my_error()' based on the diff --git a/components/masking_functions/src/masking_functions/dictionary.cpp b/components/masking_functions/src/masking_functions/dictionary.cpp index e7d1c1190511..aeb52e0c4644 100644 --- a/components/masking_functions/src/masking_functions/dictionary.cpp +++ b/components/masking_functions/src/masking_functions/dictionary.cpp @@ -26,8 +26,7 @@ namespace masking_functions { const charset_string dictionary::shared_empty{}; bool dictionary::contains(const charset_string &term) const noexcept { - // TODO: in c++20 change to terms_.contains(term) - return terms_.count(term) > 0U; + return terms_.contains(term); } const charset_string &dictionary::get_random() const noexcept { diff --git a/components/masking_functions/src/masking_functions/registration_routines.cpp b/components/masking_functions/src/masking_functions/registration_routines.cpp index bdb8a1416bdf..14b0cebffb16 100644 --- a/components/masking_functions/src/masking_functions/registration_routines.cpp +++ b/components/masking_functions/src/masking_functions/registration_routines.cpp @@ -1277,8 +1277,6 @@ DECLARE_INT_UDF_AUTO(masking_dictionary_remove) DECLARE_INT_UDF_AUTO(masking_dictionary_term_add) DECLARE_INT_UDF_AUTO(masking_dictionary_term_remove) -// TODO: in c++20 (where CTAD works for alias templates) this shoud be changed -// to 'static const udf_info_container known_udfs' // NOLINTBEGIN(cppcoreguidelines-pro-type-cstyle-cast) std::array known_udfs{DECLARE_UDF_INFO_AUTO(gen_range), DECLARE_UDF_INFO_AUTO(gen_rnd_email), From 30fd84b0431c1106b2c74f57ef10f6b5a8c9b4fc Mon Sep 17 00:00:00 2001 From: Yura Sorokin Date: Tue, 21 Jan 2025 16:11:27 +0100 Subject: [PATCH 10/11] PS-9148 feature: 8.4-specific DEBUG_SYNC improvements https://perconadev.atlassian.net/browse/PS-9148 'DBUG_EXECUTE_IF()' / 'DEBUG_SYNC()' calls reworked with 'mysql_debug_keyword_service' / 'mysql_debug_sync_service' mysql service calls (added in 8.4). --- .../masking_functions/src/component.cpp | 39 +++++++++++---- .../dictionary_flusher_thread.cpp | 50 +++++++++++++------ 2 files changed, 63 insertions(+), 26 deletions(-) diff --git a/components/masking_functions/src/component.cpp b/components/masking_functions/src/component.cpp index 37a719f8e53f..2cf31baf1506 100644 --- a/components/masking_functions/src/component.cpp +++ b/components/masking_functions/src/component.cpp @@ -24,7 +24,6 @@ #include -#include #include #include @@ -37,6 +36,10 @@ #include #include // IWYU pragma: keep #include // IWYU pragma: keep +#ifndef NDEBUG +#include +#include +#endif #include #include // IWYU pragma: keep #include @@ -46,8 +49,6 @@ #include -#include "sql/debug_sync.h" // IWYU pragma: keep - #include "masking_functions/command_service_tuple.hpp" #include "masking_functions/component_sys_variable_service_tuple.hpp" #include "masking_functions/default_sql_context_builder.hpp" @@ -101,6 +102,11 @@ REQUIRES_SERVICE_PLACEHOLDER(log_builtins_string); REQUIRES_SERVICE_PLACEHOLDER(mysql_runtime_error); +#ifndef NDEBUG +REQUIRES_SERVICE_PLACEHOLDER(mysql_debug_keyword_service); +REQUIRES_SERVICE_PLACEHOLDER(mysql_debug_sync_service); +#endif + SERVICE_TYPE(log_builtins) * log_bi; SERVICE_TYPE(log_builtins_string) * log_bs; @@ -213,13 +219,20 @@ mysql_service_status_t component_init() { masking_functions::dictionary_flusher_thread_ptr>::instance() = std::move(flusher); - // NOLINTNEXTLINE(cppcoreguidelines-avoid-do-while) - DBUG_EXECUTE_IF("enable_masking_functions_flusher_create_sync", { - MYSQL_THD extracted_thd{nullptr}; - mysql_service_mysql_current_thread_reader->get(&extracted_thd); - assert(extracted_thd != nullptr); - DEBUG_SYNC(extracted_thd, "masking_functions_after_flusher_create"); - }); + // not using 'DBUG_EXECUTE_IF()' macro from the + // 'mysql/components/util/debug_execute_if.h' and 'DEBUG_SYNC()' macro + // fromthe 'mysql/components/util/debug_sync.h' as they are not supposed + // to be used in namespaces (including anonymous) +#ifndef NDEBUG + if (mysql_service_mysql_debug_keyword_service->lookup_debug_keyword( + "enable_masking_functions_flusher_create_sync") != 0) { + MYSQL_THD current_thd{nullptr}; + mysql_service_mysql_current_thread_reader->get(¤t_thd); + assert(current_thd != nullptr); + mysql_service_mysql_debug_sync_service->debug_sync( + current_thd, "masking_functions_after_flusher_create"); + } +#endif } LogComponentErr(INFORMATION_LEVEL, ER_LOG_PRINTF_MSG, @@ -320,6 +333,12 @@ BEGIN_COMPONENT_REQUIRES(CURRENT_COMPONENT_NAME) REQUIRES_SERVICE(log_builtins_string), REQUIRES_SERVICE(mysql_runtime_error), + +#ifndef NDEBUG + REQUIRES_SERVICE(mysql_debug_keyword_service), + REQUIRES_SERVICE(mysql_debug_sync_service), +#endif + END_COMPONENT_REQUIRES(); BEGIN_COMPONENT_METADATA(CURRENT_COMPONENT_NAME) diff --git a/components/masking_functions/src/masking_functions/dictionary_flusher_thread.cpp b/components/masking_functions/src/masking_functions/dictionary_flusher_thread.cpp index 2062d176ff06..de96ec985390 100644 --- a/components/masking_functions/src/masking_functions/dictionary_flusher_thread.cpp +++ b/components/masking_functions/src/masking_functions/dictionary_flusher_thread.cpp @@ -28,7 +28,6 @@ #include #include -#include #include #include #include @@ -40,6 +39,10 @@ #include #include +#ifndef NDEBUG +#include +#include +#endif #include #include @@ -58,9 +61,11 @@ #endif #include "masking_functions/static_sql_context_builder.hpp" -#include "sql/debug_sync.h" // IWYU pragma: keep - extern REQUIRES_SERVICE_PLACEHOLDER(mysql_current_thread_reader); +#ifndef NDEBUG +extern REQUIRES_SERVICE_PLACEHOLDER(mysql_debug_keyword_service); +extern REQUIRES_SERVICE_PLACEHOLDER(mysql_debug_sync_service); +#endif namespace { @@ -294,8 +299,13 @@ void dictionary_flusher_thread::do_periodic_reload() { } cache = std::make_unique(cache_core_, sql_ctx_builder); - // NOLINTNEXTLINE(cppcoreguidelines-avoid-do-while) - DBUG_EXECUTE_IF("enable_masking_functions_flush_thread_sync", { + // not using 'DBUG_EXECUTE_IF()' macro from the + // 'mysql/components/util/debug_execute_if.h' and 'DEBUG_SYNC()' macro + // fromthe 'mysql/components/util/debug_sync.h' as they are not supposed + // to be used in namespaces (including anonymous) +#ifndef NDEBUG + if (mysql_service_mysql_debug_keyword_service->lookup_debug_keyword( + "enable_masking_functions_flush_thread_sync") != 0) { const sql_context_ptr sql_ctx{sql_ctx_builder->build()}; std::string wait_action{ "SET debug_sync = 'masking_functions_before_cache_reload WAIT_FOR " @@ -303,16 +313,18 @@ void dictionary_flusher_thread::do_periodic_reload() { std::string signal_action{ "SET debug_sync = 'masking_functions_after_cache_reload SIGNAL " "masking_functions_after_cache_reload_signal"}; - DBUG_EXECUTE_IF("enable_masking_functions_flush_thread_double_pass", { + if (mysql_service_mysql_debug_keyword_service->lookup_debug_keyword( + "enable_masking_functions_flush_thread_double_pass") != 0) { wait_action += " EXECUTE 2"; signal_action += " EXECUTE 2"; - }); + } wait_action += '\''; signal_action += '\''; sql_ctx->execute_dml(wait_action); sql_ctx->execute_dml(signal_action); - }); + } +#endif } catch (const std::exception &e) { failure_message.emplace( "Exception during flusher thread initialization - "); @@ -352,10 +364,13 @@ void dictionary_flusher_thread::do_periodic_reload() { auto expires_at{std::chrono::steady_clock::now()}; while (!is_terminated_lambda()) { if (std::chrono::steady_clock::now() >= expires_at) { - // NOLINTNEXTLINE(cppcoreguidelines-avoid-do-while) - DBUG_EXECUTE_IF("enable_masking_functions_flush_thread_sync", { - DEBUG_SYNC(extracted_thd, "masking_functions_before_cache_reload"); - }); +#ifndef NDEBUG + if (mysql_service_mysql_debug_keyword_service->lookup_debug_keyword( + "enable_masking_functions_flush_thread_sync") != 0) { + mysql_service_mysql_debug_sync_service->debug_sync( + extracted_thd, "masking_functions_before_cache_reload"); + } +#endif failure_message.reset(); try { @@ -373,10 +388,13 @@ void dictionary_flusher_thread::do_periodic_reload() { failure_message->c_str()); } - // NOLINTNEXTLINE(cppcoreguidelines-avoid-do-while) - DBUG_EXECUTE_IF("enable_masking_functions_flush_thread_sync", { - DEBUG_SYNC(extracted_thd, "masking_functions_after_cache_reload"); - }); +#ifndef NDEBUG + if (mysql_service_mysql_debug_keyword_service->lookup_debug_keyword( + "enable_masking_functions_flush_thread_sync") != 0) { + mysql_service_mysql_debug_sync_service->debug_sync( + extracted_thd, "masking_functions_after_cache_reload"); + } +#endif expires_at = std::chrono::steady_clock::now() + flush_interval_duration; } else { From 50363a97d7fb0addb116a9abcad047aa5fd00982 Mon Sep 17 00:00:00 2001 From: Yura Sorokin Date: Thu, 23 Jan 2025 00:20:46 +0100 Subject: [PATCH 11/11] Switched to clang-format-15 in CircuitCI --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2d154554a642..6bc298e9438a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,10 +13,10 @@ jobs: command: | set -o xtrace curl -sSL "http://apt.llvm.org/llvm-snapshot.gpg.key" | sudo -E apt-key add - - echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main" | sudo tee -a /etc/apt/sources.list > /dev/null + echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-15 main" | sudo tee -a /etc/apt/sources.list > /dev/null sudo -E apt-get -yq update >> ~/apt-get-update.log 2>&1 - sudo -E apt-get -yq --no-install-suggests --no-install-recommends install clang-format-10 - git diff -U0 --no-color HEAD^1 *.c *.cc *.cpp *.h *.hpp *.i *.ic *.ih | clang-format-diff-10 -style=file -p1 >_GIT_DIFF + sudo -E apt-get -yq --no-install-suggests --no-install-recommends install clang-format-15 + git diff -U0 --no-color HEAD^1 *.c *.cc *.cpp *.h *.hpp *.i *.ic *.ih | clang-format-diff-15 -style=file -p1 >_GIT_DIFF if [ ! -s _GIT_DIFF ]; then echo The last git commit is clang-formatted; else