Merge pull request #3921 from sysown/v2.x-cluster_cmp_checksums

WIP: Add checksum computation after fetch to multiple cluster modules

include/MySQL_Authentication.hpp

@@ -55,6 +55,11 @@ typedef struct _creds_group_t {
 
 class MySQL_Authentication {
 	private:
+	/**
+	 * @brief Holds the current value for 'runtime_mysql_users' used by 'ProxySQL_Admin' to reply to
+	 *  'CLUSTER_QUERY_MYSQL_USERS'.
+	 */
+	std::unique_ptr<SQLite3_result> mysql_users_resultset { nullptr };
 	creds_group_t creds_backends;
 	creds_group_t creds_frontends;
 	bool _reset(enum cred_username_type usertype);
@@ -76,6 +81,29 @@ class MySQL_Authentication {
 	bool set_SHA1(char *username, enum cred_username_type usertype, void *sha_pass);
 	unsigned int memory_usage();
 	uint64_t get_runtime_checksum();
+	/**
+	 * @brief Computes the checksum for the 'mysql_users' table contained in the supplied resultset.
+	 *  It's UNSAFE to call this function with another resultset than the specified in @param doc.
+	 * @param resultset Assumed to be the result of hte following query against the Admin interface:
+	 *   - '"SELECT username, password, active, use_ssl, default_hostgroup, default_schema,
+	 *     schema_locked, transaction_persistent, fast_forward, backend, frontend, max_connections,
+	 *     attributes, comment FROM runtime_mysql_users"'
+	 *   The order isn't relevant in the query itself because ordering is performed while processing.
+	 * @param mysql_users A 'unique_ptr' to be filled with the 'frontend' and 'backend' users found in the
+	 *  provided resulset.
+	 * @return The computed hash for the provided resultset.
+	 */
+	uint64_t get_runtime_checksum(MYSQL_RES* resultset, unique_ptr<SQLite3_result>& mysql_users);
+	/**
+	 * @brief Takes ownership of the supplied resultset and stores it in 'mysql_users_resultset' field.
+	 * @param users Holds the current value for 'runtime_mysql_users'.
+	 */
+	void save_mysql_users(std::unique_ptr<SQLite3_result>&& users);
+	/**
+	 * @brief Return a pointer to internally managed 'mysql_users_resultset' field. DO NOT FREE.
+	 * @return A pointer to the internally managed 'mysql_users_resultset'.
+	 */
+	SQLite3_result* get_current_mysql_users();
 };
 
 #endif /* __CLASS_MYSQL_AUTHENTICATION_H */

include/MySQL_HostGroups_Manager.h

@@ -54,6 +54,7 @@
 										  "lag_num_checks INT NOT NULL CHECK (lag_num_checks >= 1 AND lag_num_checks <= 16) DEFAULT 1 , comment VARCHAR ," \
 										  "UNIQUE (reader_hostgroup))"
 
+#define MYHGM_GEN_ADMIN_RUNTIME_SERVERS "SELECT hostgroup_id, hostname, port, gtid_port, CASE status WHEN 0 THEN \"ONLINE\" WHEN 1 THEN \"SHUNNED\" WHEN 2 THEN \"OFFLINE_SOFT\" WHEN 3 THEN \"OFFLINE_HARD\" WHEN 4 THEN \"SHUNNED\" END status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers ORDER BY hostgroup_id, hostname, port"
 
 typedef std::unordered_map<std::uint64_t, void *> umap_mysql_errors;
 
@@ -375,6 +376,36 @@ class MySQL_HostGroups_Manager {
 	void generate_mysql_replication_hostgroups_table();
 	Galera_Info *get_galera_node_info(int hostgroup);
 
+	/**
+	 * @brief This resultset holds the current values for 'runtime_mysql_servers' computed by either latest
+	 *  'commit' or fetched from another Cluster node. It's also used by ProxySQL_Admin to respond to the
+	 *  intercepted query 'CLUSTER_QUERY_MYSQL_SERVERS'.
+	 * @details This resultset can't right now just contain the value for 'incoming_mysql_servers' as with the
+	 *  rest of the intercepted resultset. This is due to 'runtime_mysql_servers' reconfigurations that can be
+	 *  triggered by monitoring actions like 'Galera' currently performs. These actions not only trigger status
+	 *  changes in the servers, but also re-generate the servers table via 'commit', thus generating a new
+	 *  checksum in the process. Because of this potential mismatch, the fetching server wouldn't be able to
+	 *  compute the proper checksum for the fetched 'runtime_mysql_servers' config.
+	 *
+	 *  As previously stated, these reconfigurations are monitoring actions, they can't be packed or performed
+	 *  in a single action, since monitoring data is required, which may not be already present. This makes
+	 *  this a convergent, but iterative process, that can't be compressed into a single action. Using other
+	 *  nodes 'runtime_mysql_servers' while fetching represents a best effort for avoiding these
+	 *  reconfigurations in nodes that already holds the same monitoring conditions. If monitoring
+	 *  conditions are not the same, circular fetching is still possible due to the previously described
+	 *  scenario.
+	 */
+	SQLite3_result* runtime_mysql_servers;
+	/**
+	 * @brief These resultset holds the latest values for 'incoming_*' tables used to promoted servers to runtime.
+	 * @details All these resultsets are used by 'Cluster' to fetch and promote the same configuration used in the
+	 *  node across the whole cluster. For these, the queries:
+	 *   - 'CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS'
+	 *   - 'CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS'
+	 *   - 'CLUSTER_QUERY_MYSQL_GALERA'
+	 *   - 'CLUSTER_QUERY_MYSQL_AWS_AURORA'
+	 *  Issued by 'Cluster' are intercepted by 'ProxySQL_Admin' and return the content of these resultsets.
+	 */
 	SQLite3_result *incoming_replication_hostgroups;
 
 	void generate_mysql_group_replication_hostgroups_table();
@@ -538,12 +569,31 @@ class MySQL_HostGroups_Manager {
 	void wrlock();
 	void wrunlock();
 	int servers_add(SQLite3_result *resultset);
-	bool commit();
+	bool commit(SQLite3_result* runtime_mysql_servers = nullptr, const std::string& checksum = "", const time_t epoch = 0);
+
+	/**
+	 * @brief Store the resultset for the 'runtime_mysql_servers' table set that have been loaded to runtime.
+	 *  The store configuration is later used by Cluster to propagate current config.
+	 * @param The resulset to be stored replacing the current one.
+	 */
+	void save_runtime_mysql_servers(SQLite3_result *);
+	/**
+	 * @brief These setters/getter functions store and retrieve the currently hold resultset for the
+	 *  'incoming_*' table set that have been loaded to runtime. The store configuration is later used by
+	 *  Cluster to propagate current config.
+	 * @param The resulset to be stored replacing the current one.
+	 */
+	void save_incoming_replication_hostgroups(SQLite3_result *);
+	void save_incoming_group_replication_hostgroups(SQLite3_result *);
+	void save_incoming_galera_hostgroups(SQLite3_result *);
+	void save_incoming_aws_aurora_hostgroups(SQLite3_result *);
+
+	SQLite3_result* get_current_mysql_servers_inner();
+	SQLite3_result* get_current_mysql_replication_hostgroups_inner();
+	SQLite3_result* get_current_mysql_group_replication_hostgroups_inner();
+	SQLite3_result* get_current_mysql_galera_hostgroups();
+	SQLite3_result* get_current_mysql_aws_aurora_hostgroups();
 
-	void set_incoming_replication_hostgroups(SQLite3_result *);
-	void set_incoming_group_replication_hostgroups(SQLite3_result *);
-	void set_incoming_galera_hostgroups(SQLite3_result *);
-	void set_incoming_aws_aurora_hostgroups(SQLite3_result *);
 	SQLite3_result * execute_query(char *query, char **error);
 	SQLite3_result *dump_table_mysql_servers();
 	SQLite3_result *dump_table_mysql_replication_hostgroups();

include/ProxySQL_Cluster.hpp

@@ -11,14 +11,46 @@
 
 #define PROXYSQL_NODE_METRICS_LEN	5
 
-#define CLUSTER_QUERY_MYSQL_SERVERS "SELECT hostgroup_id, hostname, port, gtid_port, status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM runtime_mysql_servers WHERE status<>'OFFLINE_HARD'"
-#define CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS "SELECT writer_hostgroup, reader_hostgroup, comment FROM runtime_mysql_replication_hostgroups"
+/**
+ * CLUSTER QUERIES DEFINITION
+ * ==========================
+ *
+ * The following queries are used by 'ProxySQL_Cluster' and intercepted by 'ProxySQL_Admin'. These queries should match
+ * the queries issued for generating the checksum for each of the target modules, for simpler reasoning, they should
+ * also represent the actual resultset being received when issuing them, since this resultset is used for computing the
+ * 'expected checksum' for the fetched config before loading it to runtime. This is done for the following modules:
+ *   - 'runtime_mysql_servers': tables 'mysql_servers', 'mysql_replication_hostgroups', 'mysql_group_replication_hostroups',
+ *     'mysql_galera_hostgroups', 'mysql_aws_aurora_hostgroups'.
+ *   - 'runtime_mysql_users'.
+ *   - 'runtime_mysql_query_rules'.
+ *
+ * IMPORTANT: For further clarify this means that it's important that the actual resultset produced by the intercepted
+ * query preserve the filtering and ordering expressed in this queries.
+ */
+
+/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_servers'. See top comment for details. */
+#define CLUSTER_QUERY_MYSQL_SERVERS "PROXY_SELECT hostgroup_id, hostname, port, gtid_port, status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM runtime_mysql_servers WHERE status<>'OFFLINE_HARD' ORDER BY hostgroup_id, hostname, port"
+
+/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_replication_hostgroups'. See top comment for details. */
+#define CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS "PROXY_SELECT writer_hostgroup, reader_hostgroup, comment FROM runtime_mysql_replication_hostgroups ORDER BY writer_hostgroup"
+
+/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_group_replication_hostgroups'. See top comment for details. */
+#define CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS "PROXY_SELECT writer_hostgroup, backup_writer_hostgroup, reader_hostgroup, offline_hostgroup, active, max_writers, writer_is_also_reader, max_transactions_behind, comment FROM runtime_mysql_group_replication_hostgroups ORDER BY writer_hostgroup"
+
+/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_aws_aurora_hostgroups'. See top comment for details. */
+#define CLUSTER_QUERY_MYSQL_AWS_AURORA "PROXY_SELECT writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, comment FROM runtime_mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup"
+
+/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_galera_hostgroups'. See top comment for details. */
+#define	CLUSTER_QUERY_MYSQL_GALERA "PROXY_SELECT writer_hostgroup, backup_writer_hostgroup, reader_hostgroup, offline_hostgroup, active, max_writers, writer_is_also_reader, max_transactions_behind, comment FROM runtime_mysql_galera_hostgroups ORDER BY writer_hostgroup"
+
+/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_users'. See top comment for details. */
+#define CLUSTER_QUERY_MYSQL_USERS "PROXY_SELECT username, password, use_ssl, default_hostgroup, default_schema, schema_locked, transaction_persistent, fast_forward, backend, frontend, max_connections, attributes, comment FROM runtime_mysql_users"
+
+/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_query_rules'. See top comment for details. */
+#define CLUSTER_QUERY_MYSQL_QUERY_RULES "PROXY_SELECT rule_id, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, cache_timeout, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, ok_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, attributes, comment FROM runtime_mysql_query_rules ORDER BY rule_id"
 
-// the following two queries are the same used in ProxySQL_Admin::load_mysql_query_rules_to_runtime() , but on runtime_ tables.
-// It is important to note that the queries in ProxySQL_Admin::load_mysql_query_rules_to_runtime() are used to compute the checksum,
-// and are the same resultset saved. Therefore it Cluster can retrieve the same resultset saved it can easily compute the checksum before loading.
-#define CLUSTER_QUERY_MYSQL_QUERY_RULES "SELECT rule_id, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, cache_timeout, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, ok_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, attributes, comment FROM runtime_mysql_query_rules ORDER BY rule_id"
-#define CLUSTER_QUERY_MYSQL_QUERY_RULES_FAST_ROUTING "SELECT username, schemaname, flagIN, destination_hostgroup, comment FROM runtime_mysql_query_rules_fast_routing ORDER BY username, schemaname, flagIN"
+/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_query_rules_fast_routing'. See top comment for details. */
+#define CLUSTER_QUERY_MYSQL_QUERY_RULES_FAST_ROUTING "PROXY_SELECT username, schemaname, flagIN, destination_hostgroup, comment FROM runtime_mysql_query_rules_fast_routing ORDER BY username, schemaname, flagIN"
 
 class ProxySQL_Checksum_Value_2: public ProxySQL_Checksum_Value {
 	public:
@@ -374,17 +406,17 @@ class ProxySQL_Cluster {
 	void p_update_metrics();
 	void thread_ending(pthread_t);
 	void join_term_thread();
-	void pull_mysql_query_rules_from_peer(const char *expected_checksum);
-	void pull_mysql_servers_from_peer();
-	void pull_mysql_users_from_peer();
+	void pull_mysql_query_rules_from_peer(const std::string& expected_checksum, const time_t epoch);
+	void pull_mysql_servers_from_peer(const std::string& expected_checksum, const time_t epoch);
+	void pull_mysql_users_from_peer(const std::string& expected_checksum, const time_t epoch);
 	/**
 	 * @brief Pulls from peer the specified global variables by the type parameter.
 	 * @param type A string specifying the type of global variables to pull from the peer, supported
 	 *  values right now are:
 	 *    - 'mysql'.
      *    - 'admin'.
 	 */
-	void pull_global_variables_from_peer(const std::string& type);
-	void pull_proxysql_servers_from_peer(const char *expected_checksum);
+	void pull_global_variables_from_peer(const std::string& type, const std::string& expected_checksum, const time_t epoch);
+	void pull_proxysql_servers_from_peer(const std::string& expected_checksum, const time_t epoch);
 };
 #endif /* CLASS_PROXYSQL_CLUSTER_H */

include/gen_utils.h

@@ -248,6 +248,6 @@ bool mywildcmp(const char *p, const char *str);
 /**
  * @brief Helper function that converts a MYSQL_RES into a 'SQLite3_result'.
  * @param resultset The resultset to be converted into a 'SQLite3_result'.
- * @return  the resulting 'SQLite3_result'.
+ * @return An 'unique_ptr' holding the resulting 'SQLite3_result'.
  */
-SQLite3_result * get_SQLite3_resulset(MYSQL_RES* resultset);
+std::unique_ptr<SQLite3_result> get_SQLite3_resulset(MYSQL_RES* resultset);