PS-9148 feature: Dictionary caching for Masking Functions Component (9.x)

.circleci/config.yml

@@ -13,10 +13,10 @@ jobs:
           command: |
             set -o xtrace
             curl -sSL "http://apt.llvm.org/llvm-snapshot.gpg.key" | sudo -E apt-key add -
-            echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main" | sudo tee -a /etc/apt/sources.list > /dev/null
+            echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-15 main" | sudo tee -a /etc/apt/sources.list > /dev/null
             sudo -E apt-get -yq update >> ~/apt-get-update.log 2>&1
-            sudo -E apt-get -yq --no-install-suggests --no-install-recommends install clang-format-10
-            git diff -U0 --no-color HEAD^1 *.c *.cc *.cpp *.h *.hpp *.i *.ic *.ih | clang-format-diff-10 -style=file -p1 >_GIT_DIFF
+            sudo -E apt-get -yq --no-install-suggests --no-install-recommends install clang-format-15
+            git diff -U0 --no-color HEAD^1 *.c *.cc *.cpp *.h *.hpp *.i *.ic *.ih | clang-format-diff-15 -style=file -p1 >_GIT_DIFF
             if [ ! -s _GIT_DIFF ]; then
                echo The last git commit is clang-formatted;
             else

components/masking_functions/CMakeLists.txt

@@ -26,27 +26,56 @@ endif()
 set(DATAMASKING_SOURCES
   src/component.cpp
 
+  src/masking_functions/bookshelf.cpp
   src/masking_functions/charset_string.cpp
   src/masking_functions/charset_string_operations.cpp
+  src/masking_functions/default_sql_context_builder.cpp
+  src/masking_functions/dictionary.cpp
+  src/masking_functions/dictionary_flusher_thread.cpp
   src/masking_functions/query_builder.cpp
+  src/masking_functions/term_cache.cpp
+  src/masking_functions/term_cache_core.cpp
   src/masking_functions/random_string_generators.cpp
   src/masking_functions/registration_routines.cpp
+  src/masking_functions/server_helpers.cpp
   src/masking_functions/sql_context.cpp
   src/masking_functions/sql_escape_functions.cpp
+  src/masking_functions/static_sql_context_builder.cpp
+  src/masking_functions/sys_vars.cpp
 
+  include/masking_functions/abstract_sql_context_builder_fwd.hpp
+  include/masking_functions/abstract_sql_context_builder.hpp
+  include/masking_functions/bookshelf_fwd.hpp
+  include/masking_functions/bookshelf.hpp
   include/masking_functions/charset_string_fwd.hpp
   include/masking_functions/charset_string.hpp
   include/masking_functions/charset_string_operations.hpp
   include/masking_functions/command_service_tuple_fwd.hpp
   include/masking_functions/command_service_tuple.hpp
+  include/masking_functions/component_sys_variable_service_tuple_fwd.hpp
+  include/masking_functions/component_sys_variable_service_tuple.hpp
+  include/masking_functions/default_sql_context_builder.hpp
+  include/masking_functions/dictionary_fwd.hpp
+  include/masking_functions/dictionary.hpp
+  include/masking_functions/dictionary_flusher_thread_fwd.hpp
+  include/masking_functions/dictionary_flusher_thread.hpp
   include/masking_functions/primitive_singleton.hpp
+  include/masking_functions/query_builder_fwd.hpp
   include/masking_functions/query_builder.hpp
+  include/masking_functions/term_cache_fwd.hpp
+  include/masking_functions/term_cache.hpp
+  include/masking_functions/term_cache_core_fwd.hpp
+  include/masking_functions/term_cache_core.hpp
   include/masking_functions/random_string_generators.hpp
   include/masking_functions/registration_routines.hpp
+  include/masking_functions/server_helpers.hpp
+  include/masking_functions/sql_context_fwd.hpp
   include/masking_functions/sql_context.hpp
   include/masking_functions/sql_escape_functions.hpp
+  include/masking_functions/static_sql_context_builder.hpp
   include/masking_functions/string_service_tuple_fwd.hpp
   include/masking_functions/string_service_tuple.hpp
+  include/masking_functions/sys_vars.hpp
 )
 
 ### Configuration ###

components/masking_functions/include/masking_functions/abstract_sql_context_builder.hpp

@@ -0,0 +1,70 @@
+/* Copyright (c) 2023 Percona LLC and/or its affiliates. All rights reserved.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software Foundation,
+   51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */
+
+#ifndef MASKING_FUNCTIONS_ABSTRACT_SQL_CONTEXT_BUILDER_HPP
+#define MASKING_FUNCTIONS_ABSTRACT_SQL_CONTEXT_BUILDER_HPP
+
+#include "masking_functions/abstract_sql_context_builder_fwd.hpp"  // IWYU pragma: export
+
+#include "masking_functions/command_service_tuple_fwd.hpp"
+#include "masking_functions/sql_context_fwd.hpp"
+
+namespace masking_functions {
+
+// This class is an abstract interface for an entity that is supposed to
+// return ready-to-use instances of 'sql_context' class.
+// This interface has two concrete implementations:
+// 1. 'default_sql_context_builder' - creates an instance of 'sql_context'
+//    class (establishes an internal connection) every time its 'build()'
+//    method is called.
+// 2. 'static_sql_context_builder' - creates an instance of 'sql_context'
+//    class (establishes an internal connection) only once in the constructor
+//    and returns a reference of this shared instance every time 'build()'
+//    method is called.
+//
+//                       abstract_sql_context_builder
+//                           ^               ^
+//                           |               |
+// default_sql_context_builder            static_sql_context_builder
+class abstract_sql_context_builder {
+ public:
+  abstract_sql_context_builder(const abstract_sql_context_builder &) = delete;
+  abstract_sql_context_builder &operator=(
+      const abstract_sql_context_builder &) = delete;
+  abstract_sql_context_builder(abstract_sql_context_builder &&) = delete;
+  abstract_sql_context_builder &operator=(abstract_sql_context_builder &&) =
+      delete;
+
+  virtual ~abstract_sql_context_builder() = default;
+
+  sql_context_ptr build() const { return do_build(); }
+
+ protected:
+  explicit abstract_sql_context_builder(const command_service_tuple &services)
+      : services_{&services} {}
+
+  const command_service_tuple &get_services() const noexcept {
+    return *services_;
+  }
+
+ private:
+  const command_service_tuple *services_;
+
+  virtual sql_context_ptr do_build() const = 0;
+};
+
+}  // namespace masking_functions
+
+#endif  // MASKING_FUNCTIONS_ABSTRACT_SQL_CONTEXT_BUILDER_HPP

components/masking_functions/include/masking_functions/abstract_sql_context_builder_fwd.hpp

@@ -0,0 +1,30 @@
+/* Copyright (c) 2023 Percona LLC and/or its affiliates. All rights reserved.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software Foundation,
+   51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */
+
+#ifndef MASKING_FUNCTIONS_ABSTRACT_SQL_CONTEXT_BUILDER_FWD_HPP
+#define MASKING_FUNCTIONS_ABSTRACT_SQL_CONTEXT_BUILDER_FWD_HPP
+
+#include <memory>
+
+namespace masking_functions {
+
+class abstract_sql_context_builder;
+
+using abstract_sql_context_builder_ptr =
+    std::shared_ptr<abstract_sql_context_builder>;
+
+}  // namespace masking_functions
+
+#endif  // MASKING_FUNCTIONS_ABSTRACT_SQL_CONTEXT_BUILDER_FWD_HPP

components/masking_functions/include/masking_functions/bookshelf.hpp

@@ -0,0 +1,63 @@
+/* Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software Foundation,
+   51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */
+
+#ifndef MASKING_FUNCTIONS_BOOKSHELF_HPP
+#define MASKING_FUNCTIONS_BOOKSHELF_HPP
+
+#include "masking_functions/bookshelf_fwd.hpp"  // IWYU pragma: export
+
+#include <map>
+
+#include "masking_functions/charset_string.hpp"
+#include "masking_functions/dictionary.hpp"
+
+namespace masking_functions {
+
+// 'bookshelf' is a collection of 'dictionary' class instances used as a
+// basic in-memory data structure for caching dictionary terms grouped
+// by dictionary. It operates on (dictionary_name, term)-pairs.
+class bookshelf {
+ public:
+  bool contains(const charset_string &dictionary_name,
+                const charset_string &term) const noexcept;
+  // returns empty charset_string if no such dictionary exist
+  const charset_string &get_random(
+      const charset_string &dictionary_name) const noexcept;
+  // returns true if there was at least one term in the 'dictionary_name'
+  // dictionary
+  // returns false if there was not a single term that belongs to the
+  // 'dictionary_name' dictionary
+  bool remove(const charset_string &dictionary_name) noexcept;
+  // returns true if the term has been successfully removed from the
+  // 'dictionary_name' dictionary
+  // returns false if the term was not present in the 'dictionary_name'
+  // dictionary
+  bool remove(const charset_string &dictionary_name,
+              const charset_string &term) noexcept;
+  // returns true if the term has been successfully inserted into the
+  // 'dictionary_name' dictionary
+  // returns false if the term is alreaddy in the 'dictionary_name'
+  // dictionary
+  bool insert(const charset_string &dictionary_name,
+              const charset_string &term);
+
+ private:
+  using dictionary_container = std::map<charset_string, dictionary>;
+  dictionary_container dictionaries_;
+};
+
+}  // namespace masking_functions
+
+#endif  // MASKING_FUNCTIONS_BOOKSHELF_HPP

components/masking_functions/include/masking_functions/bookshelf_fwd.hpp

@@ -0,0 +1,29 @@
+/* Copyright (c) 2024 Percona LLC and/or its affiliates. All rights reserved.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software Foundation,
+   51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */
+
+#ifndef MASKING_FUNCTIONS_BOOKSHELF_FWD_HPP
+#define MASKING_FUNCTIONS_BOOKSHELF_FWD_HPP
+
+#include <memory>
+
+namespace masking_functions {
+
+class bookshelf;
+
+using bookshelf_ptr = std::unique_ptr<bookshelf>;
+
+}  // namespace masking_functions
+
+#endif  // MASKING_FUNCTIONS_BOOKSHELF_HPP

components/masking_functions/include/masking_functions/charset_string.hpp

@@ -16,14 +16,14 @@
 #ifndef MASKING_FUNCTIONS_CHARSET_STRING_HPP
 #define MASKING_FUNCTIONS_CHARSET_STRING_HPP
 
+#include "masking_functions/charset_string_fwd.hpp"  // IWYU pragma: export
+
 #include <cassert>
 #include <cstdint>
 #include <memory>
 #include <string_view>
 #include <utility>
 
-#include "masking_functions/charset_string_fwd.hpp"
-
 #include "masking_functions/string_service_tuple_fwd.hpp"
 
 namespace masking_functions {
@@ -89,6 +89,8 @@ class charset_string {
 
   ~charset_string() = default;
 
+  bool is_default_constructed() const noexcept { return !impl_; }
+
   const string_service_tuple &get_services() const noexcept {
     return *impl_.get_deleter().services;
   }
@@ -130,8 +132,10 @@ class charset_string {
 
  private:
   struct deleter {
-    void operator()(void *ptr) const noexcept;
+    // NOLINTNEXTLINE(misc-non-private-member-variables-in-classes)
     const string_service_tuple *services;
+
+    void operator()(void *ptr) const noexcept;
   };
   using impl_type = std::unique_ptr<void, deleter>;
   impl_type impl_;

components/masking_functions/include/masking_functions/charset_string_fwd.hpp

@@ -16,6 +16,8 @@
 #ifndef MASKING_FUNCTIONS_CHARSET_STRING_FWD_HPP
 #define MASKING_FUNCTIONS_CHARSET_STRING_FWD_HPP
 
+#include <optional>
+
 namespace masking_functions {
 
 class charset_string;

components/masking_functions/include/masking_functions/command_service_tuple.hpp

@@ -16,12 +16,12 @@
 #ifndef MASKING_FUNCTIONS_COMMAND_SERVICE_TUPLE_HPP
 #define MASKING_FUNCTIONS_COMMAND_SERVICE_TUPLE_HPP
 
+#include "masking_functions/command_service_tuple_fwd.hpp"  // IWYU pragma: export
+
 #include <mysql/components/service.h>
 
 #include <mysql/components/services/mysql_command_services.h>
 
-#include "masking_functions/command_service_tuple_fwd.hpp"
-
 namespace masking_functions {
 
 // A set of MySQL query services required to perform a simple query
@@ -35,16 +35,22 @@ namespace masking_functions {
 //   mysql_command_query{
 //     mysql_service_mysql_command_query,
 //     mysql_service_mysql_command_query_result,
+//     mysql_service_mysql_command_field_info,
 //     mysql_service_mysql_command_options,
-//     mysql_service_mysql_command_factory
+//     mysql_service_mysql_command_factory,
+//     mysql_service_mysql_command_error_info,
+//     mysql_service_mysql_command_thread
 //   };
 // ...
 // sql_context ctx{primitive_singleton<mysql_command_query>::instance()};
 struct command_service_tuple {
   SERVICE_TYPE(mysql_command_query) * query;
   SERVICE_TYPE(mysql_command_query_result) * query_result;
+  SERVICE_TYPE(mysql_command_field_info) * field_info;
   SERVICE_TYPE(mysql_command_options) * options;
   SERVICE_TYPE(mysql_command_factory) * factory;
+  SERVICE_TYPE(mysql_command_error_info) * error_info;
+  SERVICE_TYPE(mysql_command_thread) * thread;
 };
 
 }  // namespace masking_functions