From 4c4346a71074bd6ad1a33689229d559b0907f851 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Mon, 18 Jul 2022 19:23:56 +0200 Subject: [PATCH 01/26] Add checksum computation after fetch to multiple cluster modules The following cluster modules now compute their expected checksums after fetch: - mysql_query_rules - mysql_users - mysql_servers - mysql_global_variables --- include/MySQL_Authentication.hpp | 11 + include/ProxySQL_Cluster.hpp | 21 +- lib/MySQL_Authentication.cpp | 121 ++++++++-- lib/MySQL_HostGroups_Manager.cpp | 2 +- lib/ProxySQL_Admin.cpp | 8 +- lib/ProxySQL_Cluster.cpp | 381 +++++++++++++++++++++++++------ 6 files changed, 442 insertions(+), 102 deletions(-) diff --git a/include/MySQL_Authentication.hpp b/include/MySQL_Authentication.hpp index 8602fddb81..c1d0ff134c 100644 --- a/include/MySQL_Authentication.hpp +++ b/include/MySQL_Authentication.hpp @@ -76,6 +76,17 @@ class MySQL_Authentication { bool set_SHA1(char *username, enum cred_username_type usertype, void *sha_pass); unsigned int memory_usage(); uint64_t get_runtime_checksum(); + /** + * @brief Computes the checksum for the 'mysql_users' table contained in the supplied resultset. + * It's UNSAFE to call this function with another resultset than the specified in @param doc. + * @param resultset Assumed to be the result of hte following query against the Admin interface: + * - '"SELECT username, password, active, use_ssl, default_hostgroup, default_schema, + * schema_locked, transaction_persistent, fast_forward, backend, frontend, max_connections, + * attributes, comment FROM runtime_mysql_users"' + * The order isn't relevant in the query itself because ordering is performed while processing. + * @return The computed hash for the provided resultset. + */ + uint64_t get_runtime_checksum(MYSQL_RES* resultset); }; #endif /* __CLASS_MYSQL_AUTHENTICATION_H */ diff --git a/include/ProxySQL_Cluster.hpp b/include/ProxySQL_Cluster.hpp index 86739af32f..0042693bb6 100644 --- a/include/ProxySQL_Cluster.hpp +++ b/include/ProxySQL_Cluster.hpp @@ -11,8 +11,19 @@ #define PROXYSQL_NODE_METRICS_LEN 5 -#define CLUSTER_QUERY_MYSQL_SERVERS "SELECT hostgroup_id, hostname, port, gtid_port, status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM runtime_mysql_servers WHERE status<>'OFFLINE_HARD'" -#define CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS "SELECT writer_hostgroup, reader_hostgroup, comment FROM runtime_mysql_replication_hostgroups" +/** + * @brief This query is intercepted by 'ProxySQL_Admin' and doesn't represent the actual resultset being received when + * issuing it. Instead it represent the 'intended' resultset that should be received when fetching a cluster peer + * `MYSQL_SERVERS` table. The caller issuing the query is responsable of the extra-processing. + * @details Received resultset corresponds to 'MySQL_HostGroups_Manager::dump_table_proxysql_servers' resultset. + */ +#define CLUSTER_QUERY_MYSQL_SERVERS "SELECT hostgroup_id, hostname, port, gtid_port, status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM runtime_mysql_servers WHERE status<>'OFFLINE_HARD' ORDER BY hostgroup_id, hostname, port" +/** + * @brief The result of this query is later used for the computation of the checksum of the received 'mysql_servers'. + * Since it differs from the query being issued by 'MySQL_HostGroups_Manager::commit' for the checksum computation, + * it should be revisit whenever the table 'mysql_replication_hostgroups' change. + */ +#define CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS "SELECT writer_hostgroup, reader_hostgroup, comment FROM runtime_mysql_replication_hostgroups ORDER BY writer_hostgroup" class ProxySQL_Checksum_Value_2: public ProxySQL_Checksum_Value { public: @@ -368,9 +379,9 @@ class ProxySQL_Cluster { void p_update_metrics(); void thread_ending(pthread_t); void join_term_thread(); - void pull_mysql_query_rules_from_peer(); + void pull_mysql_query_rules_from_peer(const std::string& expected_checksum); void pull_mysql_servers_from_peer(); - void pull_mysql_users_from_peer(); + void pull_mysql_users_from_peer(const std::string& expected_checksum); /** * @brief Pulls from peer the specified global variables by the type parameter. * @param type A string specifying the type of global variables to pull from the peer, supported @@ -378,7 +389,7 @@ class ProxySQL_Cluster { * - 'mysql'. * - 'admin'. */ - void pull_global_variables_from_peer(const std::string& type); + void pull_global_variables_from_peer(const std::string& type, const std::string& expected_checksum); void pull_proxysql_servers_from_peer(const char *expected_checksum); }; #endif /* CLASS_PROXYSQL_CLUSTER_H */ diff --git a/lib/MySQL_Authentication.cpp b/lib/MySQL_Authentication.cpp index 9edfe147ce..0eb8d7014f 100644 --- a/lib/MySQL_Authentication.cpp +++ b/lib/MySQL_Authentication.cpp @@ -585,44 +585,54 @@ bool MySQL_Authentication::reset() { return true; } +using std::map; -uint64_t MySQL_Authentication::_get_runtime_checksum(enum cred_username_type usertype) { - creds_group_t &cg=(usertype==USERNAME_BACKEND ? creds_backends : creds_frontends); - std::map::iterator it; - if (cg.bt_map.size() == 0) { +uint64_t compute_accounts_hash(const umap_auth& accs_map) { + if (accs_map.size() == 0) { return 0; } bool foundany = false; - SpookyHash myhash; - myhash.Init(13,4); - for (it = cg.bt_map.begin(); it != cg.bt_map.end(); ) { - account_details_t *ad=it->second; + SpookyHash acc_map_hash; + acc_map_hash.Init(13,4); + + for (const pair& map_entry : accs_map) { + const account_details_t* ad = map_entry.second; + if (ad->default_hostgroup >= 0) { foundany = true; - myhash.Update(&ad->use_ssl,sizeof(ad->use_ssl)); - myhash.Update(&ad->default_hostgroup,sizeof(ad->default_hostgroup)); - myhash.Update(&ad->schema_locked,sizeof(ad->schema_locked)); - myhash.Update(&ad->transaction_persistent,sizeof(ad->transaction_persistent)); - myhash.Update(&ad->fast_forward,sizeof(ad->fast_forward)); - myhash.Update(&ad->max_connections,sizeof(ad->max_connections)); - myhash.Update(ad->username,strlen(ad->username)); - myhash.Update(ad->password,strlen(ad->password)); + acc_map_hash.Update(&ad->use_ssl,sizeof(ad->use_ssl)); + acc_map_hash.Update(&ad->default_hostgroup,sizeof(ad->default_hostgroup)); + acc_map_hash.Update(&ad->schema_locked,sizeof(ad->schema_locked)); + acc_map_hash.Update(&ad->transaction_persistent,sizeof(ad->transaction_persistent)); + acc_map_hash.Update(&ad->fast_forward,sizeof(ad->fast_forward)); + acc_map_hash.Update(&ad->max_connections,sizeof(ad->max_connections)); + acc_map_hash.Update(ad->username,strlen(ad->username)); + acc_map_hash.Update(ad->password,strlen(ad->password)); if (ad->default_schema) - myhash.Update(ad->default_schema,strlen(ad->default_schema)); + acc_map_hash.Update(ad->default_schema,strlen(ad->default_schema)); if (ad->comment) - myhash.Update(ad->comment,strlen(ad->comment)); + acc_map_hash.Update(ad->comment,strlen(ad->comment)); if (ad->attributes) { - myhash.Update(ad->attributes,strlen(ad->attributes)); + acc_map_hash.Update(ad->attributes,strlen(ad->attributes)); } } - it++; } + if (foundany == false) { return 0; + } else { + uint64_t hash1 = 0, hash2 = 0; + acc_map_hash.Final(&hash1, &hash2); + + return hash1; } - uint64_t hash1, hash2; - myhash.Final(&hash1, &hash2); - return hash1; +} + +uint64_t MySQL_Authentication::_get_runtime_checksum(enum cred_username_type usertype) { + creds_group_t &cg=(usertype==USERNAME_BACKEND ? creds_backends : creds_frontends); + uint64_t accs_hash = compute_accounts_hash(cg.bt_map); + + return accs_hash; } uint64_t MySQL_Authentication::get_runtime_checksum() { @@ -630,3 +640,68 @@ uint64_t MySQL_Authentication::get_runtime_checksum() { uint64_t hashF = _get_runtime_checksum(USERNAME_FRONTEND); return hashB+hashF; } + +pair extract_accounts_details(MYSQL_RES* resultset) { + if (resultset == nullptr) { return { umap_auth {}, umap_auth {} }; } + + // The following order is assumed for the resulset received fields: + // - username, password, active, use_ssl, default_hostgroup, default_schema, schema_locked, + // transaction_persistent, fast_forward, backend, frontend, max_connections, attributes, comment. + umap_auth f_accs_map {}; + umap_auth b_accs_map {}; + + while (MYSQL_ROW row = mysql_fetch_row(resultset)) { + account_details_t* acc_details { new account_details_t {} }; + + acc_details->username = row[0]; + acc_details->password = row[1] ? row[1] : const_cast(""); + acc_details->__active = strcmp(row[2], "1") == 0 ? true : false; + acc_details->use_ssl = strcmp(row[3], "1") == 0 ? true : false; + acc_details->default_hostgroup = atoi(row[4]); + acc_details->default_schema = row[5] ? row[5] : const_cast(""); + acc_details->schema_locked = strcmp(row[6], "1") == 0 ? true : false; + acc_details->transaction_persistent = strcmp(row[7], "1") == 0 ? true : false; + acc_details->fast_forward = strcmp(row[8], "1") == 0 ? true : false; + acc_details->__backend = strcmp(row[9], "1") == 0 ? true : false; + acc_details->__frontend = strcmp(row[10], "1") == 0 ? true : false; + acc_details->max_connections = atoi(row[11]); + acc_details->attributes = row[12] ? row[12] : const_cast(""); + acc_details->comment = row[13] ? row[13] : const_cast(""); + + // compute the 'username' hash for the map + uint64_t u_hash = 0, _u_hash2 = 0; + SpookyHash myhash {}; + myhash.Init(1,2); + + myhash.Update(acc_details->username, strlen(acc_details->username)); + myhash.Final(&u_hash, &_u_hash2); + + if (acc_details->__backend) { + b_accs_map.insert({u_hash, acc_details}); + } else { + f_accs_map.insert({u_hash, acc_details}); + } + } + + mysql_data_seek(resultset, 0); + + return { b_accs_map, f_accs_map }; +} + +uint64_t MySQL_Authentication::get_runtime_checksum(MYSQL_RES* resultset) { + if (resultset == NULL) { return 0; } + + pair acc_maps { extract_accounts_details(resultset) }; + + uint64_t b_acc_hash = compute_accounts_hash(acc_maps.first); + uint64_t f_acc_hash = compute_accounts_hash(acc_maps.second); + + for (pair& map_entry : acc_maps.first) { + delete map_entry.second; + } + for (pair& map_entry : acc_maps.second) { + delete map_entry.second; + } + + return b_acc_hash + f_acc_hash; +} diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 3cac9df5ee..aa5f8bd68d 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -2585,7 +2585,7 @@ SQLite3_result * MySQL_HostGroups_Manager::dump_table_mysql_servers() { int cols=0; int affected_rows=0; SQLite3_result *resultset=NULL; - char *query=(char *)"SELECT hostgroup_id, hostname, port, gtid_port, weight, CASE status WHEN 0 THEN \"ONLINE\" WHEN 1 THEN \"SHUNNED\" WHEN 2 THEN \"OFFLINE_SOFT\" WHEN 3 THEN \"OFFLINE_HARD\" WHEN 4 THEN \"SHUNNED\" END, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers"; + char *query=(char *)"SELECT hostgroup_id, hostname, port, gtid_port, CASE status WHEN 0 THEN \"ONLINE\" WHEN 1 THEN \"SHUNNED\" WHEN 2 THEN \"OFFLINE_SOFT\" WHEN 3 THEN \"OFFLINE_HARD\" WHEN 4 THEN \"SHUNNED\" END status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers ORDER BY hostgroup_id, hostname, port"; proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); wrunlock(); diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 49e3402d74..a06cb24081 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -11380,8 +11380,8 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { rc=(*proxy_sqlite3_bind_text)(statement32, (idx*12)+2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*12)+3, atoi(r1->fields[2])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*12)+4, atoi(r1->fields[3])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*12)+5, ( _runtime ? r1->fields[5] : ( strcmp(r1->fields[5],"SHUNNED")==0 ? "ONLINE" : r1->fields[5] ) ), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*12)+6, atoi(r1->fields[4])); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*12)+5, ( _runtime ? r1->fields[4] : ( strcmp(r1->fields[4],"SHUNNED")==0 ? "ONLINE" : r1->fields[4] ) ), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*12)+6, atoi(r1->fields[5])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*12)+7, atoi(r1->fields[6])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*12)+8, atoi(r1->fields[7])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*12)+9, atoi(r1->fields[8])); ASSERT_SQLITE_OK(rc, admindb); @@ -11398,8 +11398,8 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { rc=(*proxy_sqlite3_bind_text)(statement1, 2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement1, 3, atoi(r1->fields[2])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement1, 4, atoi(r1->fields[3])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_text)(statement1, 5, ( _runtime ? r1->fields[5] : ( strcmp(r1->fields[5],"SHUNNED")==0 ? "ONLINE" : r1->fields[5] ) ), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 6, atoi(r1->fields[4])); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_text)(statement1, 5, ( _runtime ? r1->fields[4] : ( strcmp(r1->fields[4],"SHUNNED")==0 ? "ONLINE" : r1->fields[4] ) ), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 6, atoi(r1->fields[5])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement1, 7, atoi(r1->fields[6])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement1, 8, atoi(r1->fields[7])); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement1, 9, atoi(r1->fields[8])); ASSERT_SQLITE_OK(rc, admindb); diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index 270edf1eeb..767013cd4f 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -7,6 +7,7 @@ #include "prometheus_helpers.h" #include "ProxySQL_Cluster.hpp" +#include "MySQL_Authentication.hpp" #include "MySQL_LDAP_Authentication.hpp" #ifdef DEBUG @@ -46,6 +47,7 @@ static char *NODE_COMPUTE_DELIMITER=(char *)"-gtyw23a-"; // a random string used extern ProxySQL_Cluster * GloProxyCluster; extern ProxySQL_Admin *GloAdmin; extern MySQL_LDAP_Authentication* GloMyLdapAuth; +extern MySQL_Authentication* GloMyAuth; void * ProxySQL_Cluster_Monitor_thread(void *args) { pthread_attr_t thread_attr; @@ -598,6 +600,8 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { unsigned long long own_epoch = __sync_fetch_and_add(&GloVars.checksums_values.mysql_query_rules.epoch,0); char* own_checksum = __sync_fetch_and_add(&GloVars.checksums_values.mysql_query_rules.checksum,0); v = &checksums_values.mysql_query_rules; + const std::string v_exp_checksum { v->checksum }; + if (v->version > 1) { if ( (own_version == 1) // we just booted @@ -606,7 +610,7 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { ) { if (v->diff_check >= diff_mqr) { proxy_info("Cluster: detected a peer %s:%d with mysql_query_rules version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); - GloProxyCluster->pull_mysql_query_rules_from_peer(); + GloProxyCluster->pull_mysql_query_rules_from_peer(v_exp_checksum); if (strncmp(v->checksum, GloVars.checksums_values.mysql_query_rules.checksum, 20)==0) { // we copied from the remote server, let's also copy the same epoch GloVars.checksums_values.mysql_query_rules.epoch = v->epoch; @@ -660,6 +664,8 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { unsigned long long own_version = __sync_fetch_and_add(&GloVars.checksums_values.mysql_users.version,0); unsigned long long own_epoch = __sync_fetch_and_add(&GloVars.checksums_values.mysql_users.epoch,0); char* own_checksum = __sync_fetch_and_add(&GloVars.checksums_values.mysql_users.checksum,0); + const std::string v_exp_checksum { v->checksum }; + if (v->version > 1) { if ( (own_version == 1) // we just booted @@ -668,7 +674,7 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { ) { if (v->diff_check >= diff_mu) { proxy_info("Cluster: detected a peer %s:%d with mysql_users version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); - GloProxyCluster->pull_mysql_users_from_peer(); + GloProxyCluster->pull_mysql_users_from_peer(v_exp_checksum); if (strncmp(v->checksum, GloVars.checksums_values.mysql_users.checksum, 20)==0) { // we copied from the remote server, let's also copy the same epoch GloVars.checksums_values.mysql_users.epoch = v->epoch; @@ -691,6 +697,7 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { unsigned long long own_version = __sync_fetch_and_add(&GloVars.checksums_values.mysql_variables.version, 0); unsigned long long own_epoch = __sync_fetch_and_add(&GloVars.checksums_values.mysql_variables.epoch, 0); char* own_checksum = __sync_fetch_and_add(&GloVars.checksums_values.mysql_variables.checksum, 0); + const string expected_checksum { v->checksum }; if (v->version > 1) { if ( @@ -700,7 +707,7 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { ) { if (v->diff_check >= diff_mv) { proxy_info("Cluster: detected a peer %s:%d with mysql_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); - GloProxyCluster->pull_global_variables_from_peer("mysql"); + GloProxyCluster->pull_global_variables_from_peer("mysql", expected_checksum); if (strncmp(v->checksum, GloVars.checksums_values.mysql_variables.checksum, 20)==0) { // we copied from the remote server, let's also copy the same epoch GloVars.checksums_values.mysql_variables.epoch = v->epoch; @@ -723,6 +730,7 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { unsigned long long own_version = __sync_fetch_and_add(&GloVars.checksums_values.admin_variables.version, 0); unsigned long long own_epoch = __sync_fetch_and_add(&GloVars.checksums_values.admin_variables.epoch, 0); char* own_checksum = __sync_fetch_and_add(&GloVars.checksums_values.admin_variables.checksum, 0); + const string expected_checksum { v->checksum }; if (v->version > 1) { if ( @@ -732,7 +740,7 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { ) { if (v->diff_check >= diff_av) { proxy_info("Cluster: detected a peer %s:%d with admin_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); - GloProxyCluster->pull_global_variables_from_peer("admin"); + GloProxyCluster->pull_global_variables_from_peer("admin", expected_checksum); if (strncmp(v->checksum, GloVars.checksums_values.admin_variables.checksum, 20)==0) { // we copied from the remote server, let's also copy the same epoch GloVars.checksums_values.admin_variables.epoch = v->epoch; @@ -755,6 +763,7 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { unsigned long long own_version = __sync_fetch_and_add(&GloVars.checksums_values.ldap_variables.version, 0); unsigned long long own_epoch = __sync_fetch_and_add(&GloVars.checksums_values.ldap_variables.epoch, 0); char* own_checksum = __sync_fetch_and_add(&GloVars.checksums_values.ldap_variables.checksum, 0); + const string expected_checksum { v->checksum }; if (v->version > 1) { if ( @@ -764,7 +773,7 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { ) { if (v->diff_check >= diff_lv) { proxy_info("Cluster: detected a peer %s:%d with ldap_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); - GloProxyCluster->pull_global_variables_from_peer("ldap"); + GloProxyCluster->pull_global_variables_from_peer("ldap", expected_checksum); if (strncmp(v->checksum, GloVars.checksums_values.ldap_variables.checksum, 20)==0) { // we copied from the remote server, let's also copy the same epoch GloVars.checksums_values.ldap_variables.epoch = v->epoch; @@ -826,7 +835,55 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { } } -void ProxySQL_Cluster::pull_mysql_query_rules_from_peer() { +std::string get_checksum_from_hash(uint64_t hash) { + uint32_t d32[2] = { 0 }; + memcpy(&d32, &hash, sizeof(hash)); + + vector s_buf(20, 0); + sprintf(&s_buf[0],"0x%0X%0X", d32[0], d32[1]); + replace_checksum_zeros(&s_buf[0]); + + return string { &s_buf.front() }; +} + +/** + * @brief Computes the checksum from a MySQL resultset in the same we already do in 'SQLite3_result::raw_checksum'. + * @details For each received column computing the field length via 'strlen' is required, this is because we + * hardcode the fields length in 'MySQL_Session::SQLite3_to_MySQL'. + * @param resultset The resulset which checksum needs to be computed. + * @return The hash resulting from the checksum computation. + */ +uint64_t mysql_raw_checksum(MYSQL_RES* resultset) { + if (resultset == nullptr) { return 0; } + + uint64_t num_rows = mysql_num_rows(resultset); + if (num_rows == 0) { return 0; } + + uint32_t num_fields = mysql_num_fields(resultset); + SpookyHash myhash {}; + myhash.Init(19, 3); + + while (MYSQL_ROW row = mysql_fetch_row(resultset)) { + for (uint32_t i = 0; i < num_fields; i++) { + if (row[i]) { + // computing 'strlen' is required see @details + myhash.Update(row[i], strlen(row[i])); + } else { + myhash.Update("", 0); + } + } + } + + // restore the initial resulset index + mysql_data_seek(resultset, 0); + + uint64_t res_hash = 0, hash2 = 0; + myhash.Final(&res_hash, &hash2); + + return res_hash; +} + +void ProxySQL_Cluster::pull_mysql_query_rules_from_peer(const string& expected_checksum) { char * hostname = NULL; uint16_t port = 0; pthread_mutex_lock(&GloProxyCluster->update_mysql_query_rules_mutex); @@ -851,7 +908,7 @@ void ProxySQL_Cluster::pull_mysql_query_rules_from_peer() { //mysql_options(conn, MYSQL_OPT_READ_TIMEOUT, &timeout_long); //mysql_options(conn, MYSQL_OPT_WRITE_TIMEOUT, &timeout); { unsigned char val = 1; mysql_options(conn, MYSQL_OPT_SSL_ENFORCE, &val); } - proxy_info("Cluster: Fetching MySQL Query Rules from peer %s:%d started\n", hostname, port); + proxy_info("Cluster: Fetching MySQL Query Rules from peer %s:%d started. Expected checksum: %s\n", hostname, port, expected_checksum.c_str()); rc_conn = mysql_real_connect(conn, hostname, username, password, NULL, port, NULL, 0); if (rc_conn) { MYSQL_RES *result1 = NULL; @@ -863,6 +920,12 @@ void ProxySQL_Cluster::pull_mysql_query_rules_from_peer() { if ( rc_query == 0) { result2 = mysql_store_result(conn); proxy_info("Cluster: Fetching MySQL Query Rules from peer %s:%d completed\n", hostname, port); + + const uint64_t query_rules_hash = mysql_raw_checksum(result1) + mysql_raw_checksum(result2); + const string computed_checksum { get_checksum_from_hash(query_rules_hash) }; + + if (expected_checksum == computed_checksum) { + proxy_info("Cluster: Loading to runtime MySQL Query Rules from peer %s:%d\n", hostname, port); pthread_mutex_lock(&GloAdmin->sql_query_global_mutex); GloAdmin->admindb->execute("DELETE FROM mysql_query_rules"); @@ -962,6 +1025,14 @@ void ProxySQL_Cluster::pull_mysql_query_rules_from_peer() { } pthread_mutex_unlock(&GloAdmin->sql_query_global_mutex); metrics.p_counter_array[p_cluster_counter::pulled_mysql_query_rules_success]->Increment(); + + } else { + proxy_info( + "Cluster: Fetching MySQL Query Rules from peer %s:%d failed: Checksum changed from %s to %s\n", + hostname, port, expected_checksum.c_str(), computed_checksum.c_str() + ); + metrics.p_counter_array[p_cluster_counter::pulled_mysql_query_rules_failure]->Increment(); + } } else { proxy_info("Cluster: Fetching MySQL Query Rules from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); metrics.p_counter_array[p_cluster_counter::pulled_mysql_query_rules_failure]->Increment(); @@ -988,11 +1059,73 @@ void ProxySQL_Cluster::pull_mysql_query_rules_from_peer() { } } free(hostname); + } else { + proxy_info("No hostname found\n"); } pthread_mutex_unlock(&GloProxyCluster->update_mysql_query_rules_mutex); } -void ProxySQL_Cluster::pull_mysql_users_from_peer() { +int fetch_mysql_users_checksum(MYSQL* conn, char* hostname, uint16_t port, string& checksum) { + const char* CLUSTER_QUERY_RUNTIME_CHECKS = "SELECT checksum FROM runtime_checksums_values WHERE name='mysql_users' LIMIT 1"; + proxy_info("Cluster: Fetching checksum for MySQL Users from peer %s:%d before processing\n", hostname, port); + + int res = -1; + int my_rc = mysql_query(conn, CLUSTER_QUERY_RUNTIME_CHECKS); + if (!my_rc) { + MYSQL_RES* my_res = mysql_store_result(conn); + if (my_res != NULL && mysql_num_rows(my_res) == 1) { + MYSQL_ROW row = mysql_fetch_row(my_res); + + if (row[0] != nullptr) { + checksum = row[0]; + res = 0; + } else { + proxy_error( + "Cluster: Received empty checksum checksum for MySQL Users from peer %s:%d. Please report a bug\n", + hostname, port + ); + } + } else { + if (mysql_errno(conn)) { + proxy_info( + "Cluster: Fetching checksum for MySQL Users from peer %s:%d failed: '%s'\n", + hostname, port, mysql_error(conn) + ); + } else { + uint64_t num_rows = mysql_num_rows(my_res); + if (num_rows != 1) { + proxy_error( + "Cluster: Empty resulset fetching checksum for MySQL Users from peer %s:%d. Please report a bug\n", + hostname, port + ); + } else { + proxy_error( + "Cluster: Invalid row num '%d' fetching checksum for MySQL Users from peer %s:%d. Please report a bug\n", + num_rows, hostname, port + ); + } + } + } + } else { + proxy_info( + "Cluster: Fetching checksum for MySQL Users from peer %s:%d failed: '%s'\n", hostname, port, mysql_error(conn) + ); + } + + return res; +} + +uint64_t get_mysql_users_checksum(MYSQL_RES* resultset) { + uint64_t raw_users_checksum = GloMyAuth->get_runtime_checksum(resultset); + + if (GloMyLdapAuth) { + raw_users_checksum += GloMyLdapAuth->get_ldap_mapping_runtime_checksum(); + } + + return raw_users_checksum; +} + +void ProxySQL_Cluster::pull_mysql_users_from_peer(const string& expected_checksum) { char * hostname = NULL; uint16_t port = 0; pthread_mutex_lock(&GloProxyCluster->update_mysql_users_mutex); @@ -1017,12 +1150,19 @@ void ProxySQL_Cluster::pull_mysql_users_from_peer() { //mysql_options(conn, MYSQL_OPT_READ_TIMEOUT, &timeout_long); //mysql_options(conn, MYSQL_OPT_WRITE_TIMEOUT, &timeout); { unsigned char val = 1; mysql_options(conn, MYSQL_OPT_SSL_ENFORCE, &val); } - proxy_info("Cluster: Fetching MySQL Users from peer %s:%d started\n", hostname, port); + proxy_info("Cluster: Fetching MySQL Users from peer %s:%d started. Expected checksum: %s\n", hostname, port, expected_checksum.c_str()); rc_conn = mysql_real_connect(conn, hostname, username, password, NULL, port, NULL, 0); if (rc_conn) { rc_query = mysql_query(conn, "SELECT username, password, active, use_ssl, default_hostgroup, default_schema, schema_locked, transaction_persistent, fast_forward, backend, frontend, max_connections, attributes, comment FROM runtime_mysql_users"); if ( rc_query == 0 ) { MYSQL_RES *result = mysql_store_result(conn); + proxy_info("Cluster: Fetching MySQL Users from peer %s:%d completed\n", hostname, port); + + const uint64_t users_raw_checksum = get_mysql_users_checksum(result); + const string computed_checksum { get_checksum_from_hash(users_raw_checksum) }; + + if (expected_checksum == computed_checksum) { + GloAdmin->admindb->execute("DELETE FROM mysql_users"); MYSQL_ROW row; char *q = (char *)"INSERT INTO mysql_users (username, password, active, use_ssl, default_hostgroup, default_schema, schema_locked, transaction_persistent, fast_forward, backend, frontend, max_connections, attributes, comment) VALUES (?1 , ?2 , ?3 , ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)"; @@ -1052,7 +1192,6 @@ void ProxySQL_Cluster::pull_mysql_users_from_peer() { rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); } mysql_free_result(result); - proxy_info("Cluster: Fetching MySQL Users from peer %s:%d completed\n", hostname, port); proxy_info("Cluster: Loading to runtime MySQL Users from peer %s:%d\n", hostname, port); GloAdmin->init_users(); if (GloProxyCluster->cluster_mysql_users_save_to_disk == true) { @@ -1062,11 +1201,21 @@ void ProxySQL_Cluster::pull_mysql_users_from_peer() { proxy_info("Cluster: Saving to disk MySQL Users Rules from peer %s:%d\n", hostname, port); } metrics.p_counter_array[p_cluster_counter::pulled_mysql_users_success]->Increment(); + + metrics.p_counter_array[p_cluster_counter::pulled_mysql_users_success]->Increment(); + } else { + proxy_info( + "Cluster: Fetching MySQL Users from peer %s:%d failed: Checksum changed from %s to %s\n", + hostname, port, expected_checksum.c_str(), computed_checksum.c_str() + ); + metrics.p_counter_array[p_cluster_counter::pulled_mysql_users_failure]->Increment(); + } } else { proxy_info("Cluster: Fetching MySQL Users from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); metrics.p_counter_array[p_cluster_counter::pulled_mysql_users_failure]->Increment(); } + // TODO: Fix LDAP support for cluster synchronization if (GloMyLdapAuth) { rc_query = mysql_query( conn, @@ -1181,6 +1330,109 @@ int ProxySQL_Cluster::fetch_and_store(MYSQL* conn, const fetch_query& f_query, M return query_res; } +/** + * @brief Generates a hash for the resulset received for the query 'CLUSTER_QUERY_MYSQL_SERVERS'. + * @details Remember that this query query is intercepted by 'ProxySQL_Admin' and always answered via + * 'MySQL_HostGroups_Manager::dump_table_proxysql_servers'. + * @param resultset The resulset resulting from the mentioned query. + * @return A hash representing the contents of the resulset. + */ +uint64_t mysql_servers_raw_checksum(MYSQL_RES* resultset) { + if (resultset == nullptr) { return 0; } + + uint64_t num_rows = mysql_num_rows(resultset); + if (num_rows == 0) { return 0; } + + MYSQL_FIELD* fields = mysql_fetch_fields(resultset); + uint32_t num_fields = mysql_num_fields(resultset); + uint32_t status_idx = 0; + + for (uint32_t i = 0; i < num_fields; i++) { + if (strcmp(fields[i].name, "status") == 0) { + status_idx = i; + } + } + + SpookyHash myhash {}; + myhash.Init(19,3); + + while (MYSQL_ROW row = mysql_fetch_row(resultset)) { + for (uint32_t i = 0; i < num_fields; i++) { + if (strcmp(row[status_idx], "OFFLINE_HARD") == 0) { + continue; + } + + if (row[i]) { + if (strcmp(fields[i].name, "status") == 0) { + if (strcmp(row[i], "ONLINE") == 0 || strcmp(row[i], "SHUNNED") == 0) { + myhash.Update("0", strlen("0")); + } else { + myhash.Update("2", strlen("1")); + } + } else { + // computing 'strlen' is required see @details + myhash.Update(row[i], strlen(row[i])); + } + } else { + myhash.Update("", 0); + } + } + } + + // restore the initial resulset index + mysql_data_seek(resultset, 0); + + uint64_t res_hash = 0, hash2 = 0; + myhash.Final(&res_hash, &hash2); + + return res_hash; +} + +/** + * @brief Generates a hash from the received resultsets from executing the following queries in the specified + * order: + * - CLUSTER_QUERY_MYSQL_SERVERS. + * - CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS. + * - CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS. + * - CLUSTER_QUERY_MYSQL_GALERA. + * - CLUSTER_QUERY_MYSQL_AWS_AURORA. + * + * IMPORTANT: It's assumed that the previous queries were successful and that the resultsets are received in + * the specified order. + * @param results The resultsets from whose to compute the checksum. Previous described order is required. + * @return Zero if the received resultset were empty, the computed hash otherwise. + */ +uint64_t compute_servers_tables_raw_checksum(const vector& results) { + bool init = false; + SpookyHash myhash {}; + + for (size_t i = 0; i < results.size(); i++) { + uint64_t raw_hash = 0; + + if (i == 0) { + raw_hash = mysql_servers_raw_checksum(results[i]); + } else { + raw_hash = mysql_raw_checksum(results[i]); + } + + if (raw_hash != 0) { + if (init == false) { + init = true; + myhash.Init(19, 3); + } + + myhash.Update(&raw_hash, sizeof(raw_hash)); + } + } + + uint64_t servers_hash = 0, _hash2 = 0; + if (init) { + myhash.Final(&servers_hash, &_hash2); + } + + return servers_hash; +} + void ProxySQL_Cluster::pull_mysql_servers_from_peer() { char * hostname = NULL; uint16_t port = 0; @@ -1219,7 +1471,7 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer() { // group_replication_hostgroups query and messages const char* CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS = "SELECT writer_hostgroup, backup_writer_hostgroup, reader_hostgroup, offline_hostgroup, active, " - "max_writers, writer_is_also_reader, max_transactions_behind, comment FROM runtime_mysql_group_replication_hostgroups"; + "max_writers, writer_is_also_reader, max_transactions_behind, comment FROM runtime_mysql_group_replication_hostgroups ORDER BY writer_hostgroup"; std::string fetch_group_replication_hostgroups = ""; string_format("Cluster: Fetching 'MySQL Group Replication Hostgroups' from peer %s:%d\n", fetch_group_replication_hostgroups, hostname, port); std::string fetch_group_replication_hostgroups_err = ""; @@ -1228,7 +1480,7 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer() { // AWS Aurora query and messages const char* CLUSTER_QUERY_MYSQL_AWS_AURORA = "SELECT writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, " - "check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, comment FROM runtime_mysql_aws_aurora_hostgroups"; + "check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, comment FROM runtime_mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup"; std::string fetch_aws_aurora_start = ""; string_format("Cluster: Fetching 'MySQL Aurora Hostgroups' from peer %s:%d\n", fetch_aws_aurora_start, hostname, port); std::string fetch_aws_aurora_err = ""; @@ -1237,20 +1489,19 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer() { // Galera query and messages const char* CLUSTER_QUERY_MYSQL_GALERA = "SELECT writer_hostgroup, backup_writer_hostgroup, reader_hostgroup, offline_hostgroup, active, " - "max_writers, writer_is_also_reader, max_transactions_behind, comment FROM runtime_mysql_galera_hostgroups"; + "max_writers, writer_is_also_reader, max_transactions_behind, comment FROM runtime_mysql_galera_hostgroups ORDER BY writer_hostgroup"; std::string fetch_galera_start = ""; string_format("Cluster: Fetching 'MySQL Galera Hostgroups' from peer %s:%d\n", fetch_galera_start, hostname, port); std::string fetch_galera_err = ""; string_format("Cluster: Fetching 'MySQL Galera Hostgroups' from peer %s:%d failed: \n", fetch_galera_err, hostname, port); - // Checksums query and messages - const char* CLUSTER_QUERY_RUNTIME_CHECKS = "SELECT * FROM runtime_checksums_values WHERE name='mysql_servers' LIMIT 1"; - std::string fetch_checksums_start = ""; - string_format("Cluster: Fetching checksum for MySQL Servers from peer %s:%d before proceessing\n", fetch_checksums_start, hostname, port); - std::string fetch_checksums_err = ""; - string_format("Cluster: Fetching checksum for MySQL Servers from peer %s:%d failed: \n", fetch_checksums_err, hostname, port); - // Create fetching queries + + /** + * @brief Array of queries definitions used to fetch data from a peer. + * @details All the queries defined here require to be updated if their target table definition is + * changed. More details on 'CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS' definition. + */ fetch_query queries[] = { { CLUSTER_QUERY_MYSQL_SERVERS, @@ -1280,12 +1531,6 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer() { p_cluster_counter::pulled_mysql_servers_aws_aurora_hostgroups_success, p_cluster_counter::pulled_mysql_servers_aws_aurora_hostgroups_failure, { fetch_aws_aurora_start, "", fetch_aws_aurora_err } - }, - { - CLUSTER_QUERY_RUNTIME_CHECKS, - p_cluster_counter::pulled_mysql_servers_runtime_checks_success, - p_cluster_counter::pulled_mysql_servers_runtime_checks_failure, - { fetch_checksums_start, "", fetch_checksums_err } } }; @@ -1303,26 +1548,18 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer() { } if (fetching_error == false) { - MYSQL_ROW row; - char *checks = NULL; - while ((row = mysql_fetch_row(results[5]))) { - if (checks) { // health check - free(checks); - checks = NULL; - } - if (row[3]) { - checks = strdup(row[3]); // checksum - } - } - if (checks && strcmp(checks,peer_checksum)==0) { + const uint64_t servers_hash = compute_servers_tables_raw_checksum(results); + const string computed_checksum { get_checksum_from_hash(servers_hash) }; + + if (computed_checksum == peer_checksum) { // we are OK to sync! - proxy_info("Cluster: Fetching checksum for MySQL Servers from peer %s:%d successful. Checksum: %s\n", hostname, port, checks); + proxy_info("Cluster: Fetching checksum for 'MySQL Servers' from peer %s:%d successful. Checksum: %s\n", hostname, port, computed_checksum.c_str()); // sync mysql_servers proxy_info("Cluster: Writing mysql_servers table\n"); GloAdmin->mysql_servers_wrlock(); GloAdmin->admindb->execute("DELETE FROM mysql_servers"); MYSQL_ROW row; - char *q=(char *)"INSERT INTO mysql_servers (hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment) VALUES (%s, \"%s\", %s, %s, %s, \"%s\", %s, %s, %s, %s, %s, '%s')"; + char *q=(char *)"INSERT INTO mysql_servers (hostgroup_id, hostname, port, gtid_port, status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment) VALUES (%s, \"%s\", %s, %s, \"%s\", %s, %s, %s, %s, %s, %s, '%s')"; while ((row = mysql_fetch_row(results[0]))) { int i; int l=0; @@ -1332,7 +1569,7 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer() { char *o=escape_string_single_quotes(row[11],false); char *query = (char *)malloc(strlen(q)+i+strlen(o)+64); - sprintf(query,q,row[0],row[1],row[2],row[3], row[4], ( strcmp(row[5],"SHUNNED")==0 ? "ONLINE" : row[5] ), row[6],row[7],row[8],row[9],row[10],o); + sprintf(query,q,row[0],row[1],row[2],row[3], ( strcmp(row[4],"SHUNNED")==0 ? "ONLINE" : row[4] ), row[5], row[6],row[7],row[8],row[9],row[10],o); if (o!=row[11]) { // there was a copy free(o); } @@ -1496,6 +1733,12 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer() { proxy_info("Cluster: Not saving to disk MySQL Servers from peer %s:%d failed.\n", hostname, port); } GloAdmin->mysql_servers_wrunlock(); + } else { + proxy_info( + "Cluster: Fetching MySQL Servers from peer %s:%d failed: Checksum changed from %s to %s\n", + hostname, port, peer_checksum, computed_checksum.c_str() + ); + metrics.p_counter_array[p_cluster_counter::pulled_mysql_variables_failure]->Increment(); } // free results @@ -1521,7 +1764,7 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer() { pthread_mutex_unlock(&GloProxyCluster->update_mysql_servers_mutex); } -void ProxySQL_Cluster::pull_global_variables_from_peer(const std::string& var_type) { +void ProxySQL_Cluster::pull_global_variables_from_peer(const string& var_type, const string& expected_checksum) { char * hostname = NULL; uint16_t port = 0; char* vars_type_str = nullptr; @@ -1594,10 +1837,18 @@ void ProxySQL_Cluster::pull_global_variables_from_peer(const std::string& var_ty s_query += " AND variable_name NOT IN " + string(CLUSTER_SYNC_INTERFACES_MYSQL); } } + s_query += " ORDER BY variable_name"; mysql_query(conn, s_query.c_str()); if (rc_query == 0) { MYSQL_RES *result = mysql_store_result(conn); + proxy_info("Cluster: Fetching %s Variables from peer %s:%d completed\n", vars_type_str, hostname, port); + + uint64_t glovars_hash = mysql_raw_checksum(result); + string computed_checksum { get_checksum_from_hash(glovars_hash) }; + + if (expected_checksum == computed_checksum) { + std::string d_query = ""; // remember that we read from runtime_global_variables but write into global_variables string_format("DELETE FROM global_variables WHERE variable_name LIKE '%s-%%'", d_query, var_type.c_str()); @@ -1629,7 +1880,6 @@ void ProxySQL_Cluster::pull_global_variables_from_peer(const std::string& var_ty } mysql_free_result(result); - proxy_info("Cluster: Fetching %s Variables from peer %s:%d completed\n", vars_type_str, hostname, port); proxy_info("Cluster: Loading to runtime %s Variables from peer %s:%d\n", vars_type_str, hostname, port); if (var_type == "mysql") { @@ -1658,6 +1908,14 @@ void ProxySQL_Cluster::pull_global_variables_from_peer(const std::string& var_ty assert(0); } metrics.p_counter_array[success_metric]->Increment(); + + } else { + proxy_info( + "Cluster: Fetching %s Variables from peer %s:%d failed: Checksum changed from %s to %s\n", + vars_type_str, hostname, port, expected_checksum.c_str(), computed_checksum.c_str() + ); + metrics.p_counter_array[p_cluster_counter::pulled_mysql_variables_failure]->Increment(); + } } else { proxy_info("Cluster: Fetching %s Variables from peer %s:%d failed: %s\n", vars_type_str, hostname, port, mysql_error(conn)); metrics.p_counter_array[failure_metric]->Increment(); @@ -1707,34 +1965,16 @@ void ProxySQL_Cluster::pull_proxysql_servers_from_peer(const char *expected_chec if (rc_conn) { rc_query = mysql_query(conn,"SELECT hostname, port, weight, comment FROM runtime_proxysql_servers ORDER BY hostname, port"); if ( rc_query == 0 ) { - char **pta=(char **)malloc(sizeof(char *)*4); - SQLite3_result *result3=new SQLite3_result(4); - result3->add_column_definition(SQLITE_TEXT,"hostname"); - result3->add_column_definition(SQLITE_TEXT,"port"); - result3->add_column_definition(SQLITE_TEXT,"weight"); - result3->add_column_definition(SQLITE_TEXT,"comment"); - MYSQL_RES *result = mysql_store_result(conn); - MYSQL_ROW row; - while ((row = mysql_fetch_row(result))) { - for (int i=0; i<4; i++) { - pta[i] = row[i]; - } - result3->add_row(pta); - } - free(pta); - uint64_t hash1 = result3->raw_checksum(); - uint32_t d32[2]; - char buf[20] = { 0 }; - memcpy(&d32, &hash1, sizeof(hash1)); - sprintf(buf,"0x%0X%0X", d32[0], d32[1]); - replace_checksum_zeros(buf); - delete result3; - proxy_info("Cluster: Fetching ProxySQL Servers from peer %s:%d completed. Computed checksum: %s\n", hostname, port, buf); - if (strcmp(buf, expected_checksum)==0) { + MYSQL_RES* result = mysql_store_result(conn); + uint64_t proxy_servers_hash = mysql_raw_checksum(result); + const string computed_cks { get_checksum_from_hash(proxy_servers_hash) }; + proxy_info("Cluster: Fetching ProxySQL Servers from peer %s:%d completed. Computed checksum: %s\n", hostname, port, computed_cks.c_str()); + + if (computed_cks == expected_checksum) { mysql_data_seek(result,0); GloAdmin->admindb->execute("DELETE FROM proxysql_servers"); char *q=(char *)"INSERT INTO proxysql_servers (hostname, port, weight, comment) VALUES (\"%s\", %s, %s, '%s')"; - while ((row = mysql_fetch_row(result))) { + while (MYSQL_ROW row = mysql_fetch_row(result)) { int i; int l=0; for (i=0; i<3; i++) { @@ -1769,7 +2009,10 @@ void ProxySQL_Cluster::pull_proxysql_servers_from_peer(const char *expected_chec } metrics.p_counter_array[p_cluster_counter::pulled_proxysql_servers_success]->Increment(); } else { - proxy_info("Cluster: Fetching ProxySQL Servers from peer %s:%d failed: Checksum changed from %s to %s\n", hostname, port, expected_checksum, buf); + proxy_info( + "Cluster: Fetching ProxySQL Servers from peer %s:%d failed: Checksum changed from %s to %s\n", + hostname, port, expected_checksum, computed_cks.c_str() + ); metrics.p_counter_array[p_cluster_counter::pulled_proxysql_servers_failure]->Increment(); } mysql_free_result(result); From 4119c441606e0951cf6a6cf4ea0071505737aece Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Mon, 18 Jul 2022 19:35:32 +0200 Subject: [PATCH 02/26] Fix potential race conditions while generating 'gtid_executed_tables' --- lib/MySQL_HostGroups_Manager.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index aa5f8bd68d..66004cebaf 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -2124,6 +2124,9 @@ void MySQL_HostGroups_Manager::generate_mysql_gtid_executed_tables() { it++; } + // NOTE: We are required to lock while iterating over 'MyHostGroups'. Otherwise race conditions could take place, + // e.g. servers could be purged by 'purge_mysql_servers_table' and invalid memory be accessed. + wrlock(); for (unsigned int i=0; ilen; i++) { MyHGC *myhgc=(MyHGC *)MyHostGroups->index(i); MySrvC *mysrvc=NULL; @@ -2164,6 +2167,7 @@ void MySQL_HostGroups_Manager::generate_mysql_gtid_executed_tables() { } } } + wrunlock(); std::vector to_remove; it = gtid_map.begin(); while(it != gtid_map.end()) { From 63e8c5730918b5e7d4dda7b08b575e9ee568a29d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Mon, 18 Jul 2022 19:36:36 +0200 Subject: [PATCH 03/26] Fix invalid memory accesses in Aurora monitoring Invalid memory accesses could take place if no servers can be pinged. --- lib/MySQL_Monitor.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 6527c54cae..ab8f8d8755 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -4294,6 +4294,11 @@ void * monitor_AWS_Aurora_thread_HG(void *arg) { cur_host_idx++; } } + // NOTE: 'cur_host_idx' should never be higher than 'num_hosts' otherwise later an invalid memory access + // can table place later when accessing 'hpa[cur_host_idx]'. + if (cur_host_idx >= num_hosts) { + cur_host_idx = num_hosts - 1; + } pthread_mutex_unlock(&GloMyMon->aws_aurora_mutex); bool exit_now = false; From 7286b334d1c5b350d7432bdac01b886f97a58a58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Mon, 18 Jul 2022 19:41:19 +0200 Subject: [PATCH 04/26] Add utility function to convert 'MYSQL_RES' into 'SQLite3_result' --- include/gen_utils.h | 11 +++++++++++ lib/gen_utils.cpp | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/include/gen_utils.h b/include/gen_utils.h index 21b5b977e2..507114f09e 100644 --- a/include/gen_utils.h +++ b/include/gen_utils.h @@ -1,6 +1,10 @@ #ifndef __CLASS_PTR_ARRAY_H #define __CLASS_PTR_ARRAY_H + +#include + #include "proxysql.h" +#include "sqlite3db.h" #define MIN_ARRAY_LEN 8 #define MIN_ARRAY_DELETE_RATIO 8 @@ -240,3 +244,10 @@ int remove_spaces(const char *); char *trim_spaces_in_place(char *str); char *trim_spaces_and_quotes_in_place(char *str); bool mywildcmp(const char *p, const char *str); + +/** + * @brief Helper function that converts a MYSQL_RES into a 'SQLite3_result'. + * @param resultset The resultset to be converted into a 'SQLite3_result'. + * @return An 'unique_ptr' holding the resulting 'SQLite3_result'. + */ +std::unique_ptr get_SQLite3_resulset(MYSQL_RES* resultset); diff --git a/lib/gen_utils.cpp b/lib/gen_utils.cpp index 0f7edf8c53..6ba4c2802b 100644 --- a/lib/gen_utils.cpp +++ b/lib/gen_utils.cpp @@ -1,5 +1,11 @@ +#include + #include "gen_utils.h" + +using std::vector; +using std::unique_ptr; + char *escape_string_single_quotes(char *input, bool free_it) { int i,j,l; char *o=NULL; // output string, if any @@ -220,3 +226,30 @@ bool Proxy_file_regular(const char *path) { return false; } +std::unique_ptr get_SQLite3_resulset(MYSQL_RES* resultset) { + if (resultset == nullptr) { + return std::unique_ptr(nullptr); + } + + uint32_t num_fields = mysql_num_fields(resultset); + MYSQL_FIELD* fields = mysql_fetch_fields(resultset); + + std::unique_ptr sqlite_result { new SQLite3_result(num_fields) }; + + for (uint32_t i = 0; i < num_fields; i++) { + sqlite_result->add_column_definition(SQLITE_TEXT, fields[i].name); + } + + vector pta(static_cast(num_fields)); + while (MYSQL_ROW row = mysql_fetch_row(resultset)) { + for (uint32_t i = 0; i < num_fields; i++) { + pta[i] = row[i]; + } + sqlite_result->add_row(&pta[0]); + } + + // restore the initial resulset index + mysql_data_seek(resultset, 0); + + return sqlite_result; +} From 134448f0587a7452ed63aa8f4e77474c3f9737d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Mon, 18 Jul 2022 21:34:49 +0200 Subject: [PATCH 05/26] Improve 'test_cluster_sync' including 'mysql_servers' --- test/tap/tap/utils.cpp | 48 +++++ test/tap/tap/utils.h | 18 ++ test/tap/tests/test_cluster_sync-t.cpp | 194 +++++++++++++++++- test/tap/tests/test_unsupported_queries-t.cpp | 64 ------ 4 files changed, 258 insertions(+), 66 deletions(-) diff --git a/test/tap/tap/utils.cpp b/test/tap/tap/utils.cpp index a2b60ed6a4..60f0f5d0a2 100644 --- a/test/tap/tap/utils.cpp +++ b/test/tap/tap/utils.cpp @@ -692,3 +692,51 @@ MYSQL* wait_for_proxysql(const conn_opts_t& opts, int timeout) { return admin; } } + +int get_variable_value( + MYSQL* proxysql_admin, const std::string& variable_name, std::string& variable_value, bool runtime +) { + if (proxysql_admin == NULL) { + return EINVAL; + } + + int res = EXIT_FAILURE; + + const std::string t_select_var_query { + "SELECT * FROM %sglobal_variables WHERE Variable_name='%s'" + }; + std::string select_var_query {}; + + if (runtime) { + string_format(t_select_var_query, select_var_query, "runtime_", variable_name.c_str()); + } else { + string_format(t_select_var_query, select_var_query, "", variable_name.c_str()); + } + + MYSQL_QUERY(proxysql_admin, select_var_query.c_str()); + + MYSQL_RES* admin_res = mysql_store_result(proxysql_admin); + if (!admin_res) { + diag("'mysql_store_result' at line %d failed: %s", __LINE__, mysql_error(proxysql_admin)); + goto cleanup; + } + + { + MYSQL_ROW row = mysql_fetch_row(admin_res); + if (!row || row[0] == nullptr || row[1] == nullptr) { + diag("'mysql_fetch_row' at line %d returned 'NULL'", __LINE__); + res = -1; + goto cleanup; + } + + // Extract the result + std::string _variable_value { row[1] }; + variable_value = _variable_value; + + res = EXIT_SUCCESS; + } + +cleanup: + + return res; +} diff --git a/test/tap/tap/utils.h b/test/tap/tap/utils.h index d6effc346c..f9c67a1059 100644 --- a/test/tap/tap/utils.h +++ b/test/tap/tap/utils.h @@ -232,4 +232,22 @@ struct conn_opts_t { */ MYSQL* wait_for_proxysql(const conn_opts_t& opts, int timeout); +/** + * @brief Extract the current value for a given 'variable_name' from + * ProxySQL current configuration, either MEMORY or RUNTIME. + * @param proxysql_admin An already opened connection to ProxySQL Admin. + * @param variable_name The name of the variable to be retrieved from ProxySQL + * config. + * @param variable_value Reference to string acting as output parameter which + * will content the value of the specified variable. + * @return EXIT_SUCCESS, or one of the following error codes: + * - EINVAL if supplied 'proxysql_admin' is NULL. + * - '-1' in case of ProxySQL returns an 'NULL' row for the query selecting + * the variable 'sqliteserver-read_only'. + * - EXIT_FAILURE in case other operation failed. + */ +int get_variable_value( + MYSQL* proxysql_admin, const std::string& variable_name, std::string& variable_value, bool runtime=false +); + #endif // #define UTILS_H diff --git a/test/tap/tests/test_cluster_sync-t.cpp b/test/tap/tests/test_cluster_sync-t.cpp index 385f7e85f2..807f8b63b5 100644 --- a/test/tap/tests/test_cluster_sync-t.cpp +++ b/test/tap/tests/test_cluster_sync-t.cpp @@ -2,6 +2,7 @@ * @file test_cluster_sync-t.cpp * @brief Checks that ProxySQL is properly syncing multiple elements from another Cluster instance. * @details Checks the sync of the following tables: + * - 'mysql_servers' with different status to check expected propagation. * - 'mysql_galera_hostgroups' with and without NULL comments. * - 'mysql_group_replication_hostgroups' with and without NULL comments. * - 'proxysql_servers' with new values and empty (exercising bug from '#3847'). @@ -60,6 +61,7 @@ using std::string; using std::vector; +using std::tuple; /** * @brief Helper function to verify that the sync of a table (or variable) have been performed. @@ -229,6 +231,166 @@ int check_nodes_sync( return EXIT_SUCCESS; } +const std::string t_debug_query = "mysql -u%s -p%s -h %s -P%d -C -e \"%s\""; + +using mysql_server_tuple = tuple; + + +int check_mysql_servers_sync( + const CommandLine& cl, MYSQL* proxy_admin, MYSQL* r_proxy_admin, + const vector& insert_mysql_servers_values +) { + MYSQL_QUERY(proxy_admin, "SET mysql-monitor_enabled='false'"); + MYSQL_QUERY(proxy_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); + + // TODO: Perform a wait for the values of 'mysql-monitor_read_only_interval' and 'mysql-monitor_read_only_timeout'. + // This will ensure no further monitor 'read-only' operations are being performed before proceeding. + string monitor_read_only_interval {}; + int g_err = get_variable_value(proxy_admin, "mysql-monitor_read_only_interval", monitor_read_only_interval); + if (g_err) { return EXIT_FAILURE; } + string monitor_read_only_timeout {}; + g_err = get_variable_value(proxy_admin, "mysql-monitor_read_only_timeout", monitor_read_only_timeout); + if (g_err) { return EXIT_FAILURE; } + + uint64_t wait = std::stol(monitor_read_only_interval) / 1000 + std::stol(monitor_read_only_timeout) / 1000; + sleep(wait*2); + + std::string print_master_mysql_servers_hostgroups = ""; + string_format(t_debug_query, print_master_mysql_servers_hostgroups, cl.admin_username, cl.admin_password, cl.host, cl.admin_port, "SELECT * FROM runtime_mysql_servers"); + std::string print_replica_mysql_servers_hostgroups = ""; + string_format(t_debug_query, print_replica_mysql_servers_hostgroups, "radmin", "radmin", cl.host, R_PORT, "SELECT * FROM mysql_servers"); + + // Configure 'mysql_servers' and check sync with NULL comments + const char* t_insert_mysql_servers = + "INSERT INTO mysql_servers (" + " hostgroup_id, hostname, port, gtid_port, status, weight, compression, max_connections," + " max_replication_lag, use_ssl, max_latency_ms, comment" + ") VALUES (%d, '%s', %d, %d, '%s', %d, %d, %d, %d, %d, %d, '%s')"; + std::vector insert_mysql_mysql_servers_hostgroup_queries {}; + + for (auto const& values : insert_mysql_servers_values) { + std::string insert_mysql_servers_hostgroup_query = ""; + string_format( + t_insert_mysql_servers, + insert_mysql_servers_hostgroup_query, + std::get<0>(values), + std::get<1>(values).c_str(), + std::get<2>(values), + std::get<3>(values), + std::get<4>(values).c_str(), + std::get<5>(values), + std::get<6>(values), + std::get<7>(values), + std::get<8>(values), + std::get<9>(values), + std::get<10>(values), + std::get<11>(values).c_str() + ); + insert_mysql_mysql_servers_hostgroup_queries.push_back(insert_mysql_servers_hostgroup_query); + } + + std::vector select_mysql_mysql_servers_hostgroup_queries {}; + + for (auto const& values : insert_mysql_servers_values) { + const char* t_select_mysql_servers_inserted_entries = + "SELECT COUNT(*) FROM mysql_servers WHERE hostgroup_id=%d AND hostname='%s'" + " AND port=%d AND gtid_port=%d AND status='%s' AND weight=%d AND" + " compression=%d AND max_connections=%d AND max_replication_lag=%d" + " AND use_ssl=%d AND max_latency_ms=%d AND comment='%s'"; + + string status { std::get<4>(values) }; + if (status == "SHUNNED") { status = "ONLINE"; } + std::string select_mysql_servers_hostgroup_query = ""; + + string_format( + t_select_mysql_servers_inserted_entries, + select_mysql_servers_hostgroup_query, + std::get<0>(values), + std::get<1>(values).c_str(), + std::get<2>(values), + std::get<3>(values), + status.c_str(), + std::get<5>(values), + std::get<6>(values), + std::get<7>(values), + std::get<8>(values), + std::get<9>(values), + std::get<10>(values), + std::get<11>(values).c_str() + ); + + if (status == "OFFLINE_HARD") { + t_select_mysql_servers_inserted_entries = "SELECT NOT(%s)"; + string_format( + t_select_mysql_servers_inserted_entries, + select_mysql_servers_hostgroup_query, + select_mysql_servers_hostgroup_query.c_str() + ); + } + + select_mysql_mysql_servers_hostgroup_queries.push_back(select_mysql_servers_hostgroup_query); + } + + // SETUP CONFIG + + // Backup current table + MYSQL_QUERY(proxy_admin, "CREATE TABLE mysql_servers_sync_test_2687 AS SELECT * FROM mysql_servers"); + MYSQL_QUERY(proxy_admin, "DELETE FROM mysql_servers"); + + // Insert the new mysql_servers hostgroups values + for (const auto& query : insert_mysql_mysql_servers_hostgroup_queries) { + MYSQL_QUERY(proxy_admin, query.c_str()); + } + MYSQL_QUERY(proxy_admin, "LOAD MYSQL SERVERS TO RUNTIME"); + + std::cout << "MASTER TABLE BEFORE SYNC:" << std::endl; + system(print_master_mysql_servers_hostgroups.c_str()); + + // SYNCH CHECK + + // Sleep until timeout waiting for synchronization + uint waited = 0; + bool not_synced_query = false; + while (waited < SYNC_TIMEOUT) { + not_synced_query = false; + // Check that all the entries have been synced + for (const auto& query : select_mysql_mysql_servers_hostgroup_queries) { + MYSQL_QUERY(r_proxy_admin, query.c_str()); + MYSQL_RES* mysql_servers_res = mysql_store_result(r_proxy_admin); + MYSQL_ROW row = mysql_fetch_row(mysql_servers_res); + int row_value = atoi(row[0]); + mysql_free_result(mysql_servers_res); + + if (row_value == 0) { + not_synced_query = true; + diag("Waiting on query '%s'...\n", query.c_str()); + break; + } + } + + if (not_synced_query) { + waited += 1; + sleep(1); + } else { + break; + } + } + + std::cout << "REPLICA TABLE AFTER SYNC:" << std::endl; + system(print_replica_mysql_servers_hostgroups.c_str()); + ok(not_synced_query == false, "'mysql_servers' with NULL comments should be synced."); + + // TEARDOWN CONFIG + MYSQL_QUERY(proxy_admin, "DELETE FROM mysql_servers"); + MYSQL_QUERY(proxy_admin, "INSERT INTO mysql_servers SELECT * FROM mysql_servers_sync_test_2687"); + MYSQL_QUERY(proxy_admin, "DROP TABLE mysql_servers_sync_test_2687"); + MYSQL_QUERY(proxy_admin, "LOAD MYSQL SERVERS TO RUNTIME"); + MYSQL_QUERY(proxy_admin, "SET mysql-monitor_enabled='true'"); + MYSQL_QUERY(proxy_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); + + return EXIT_SUCCESS; +} + int main(int, char**) { int res = 0; CommandLine cl; @@ -239,10 +401,9 @@ int main(int, char**) { return EXIT_FAILURE; } - plan(12); + plan(15); const std::string fmt_config_file = std::string(cl.workdir) + "test_cluster_sync_config/test_cluster_sync.cnf"; - const std::string t_debug_query = "mysql -u%s -p%s -h %s -P%d -C -e \"%s\""; MYSQL* proxy_admin = mysql_init(NULL); @@ -354,6 +515,35 @@ int main(int, char**) { sleep(2); + { + vector insert_mysql_servers_values { + std::make_tuple(0, "127.0.0.1", 13306, 12, "ONLINE", 1, 1, 1000, 300, 1, 200, ""), + std::make_tuple(1, "127.0.0.1", 13307, 13, "OFFLINE_SOFT", 2, 1, 500, 300, 1, 200, ""), + std::make_tuple(2, "127.0.0.1", 13308, 14, "OFFLINE_HARD", 2, 1, 500, 300, 1, 200, ""), + std::make_tuple(3, "127.0.0.1", 13309, 15, "SHUNNED", 1, 0, 500, 300, 1, 200, "") + }; + + check_mysql_servers_sync(cl, proxy_admin, r_proxy_admin, insert_mysql_servers_values); + + vector insert_mysql_servers_values_2 { + std::make_tuple(0, "127.0.0.1", 13306, 12, "ONLINE", 1, 1, 1000, 300, 1, 200, "mysql_1"), + std::make_tuple(1, "127.0.0.1", 13307, 13, "OFFLINE_SOFT", 2, 1, 500, 300, 1, 200, "mysql_2_offline"), + std::make_tuple(2, "127.0.0.1", 13308, 14, "OFFLINE_SOFT", 2, 1, 500, 300, 1, 200, "mysql_3_offline"), + std::make_tuple(3, "127.0.0.1", 13309, 15, "OFFLINE_SOFT", 1, 0, 500, 300, 1, 200, "mysql_4_offline") + }; + + check_mysql_servers_sync(cl, proxy_admin, r_proxy_admin, insert_mysql_servers_values_2); + + vector insert_mysql_servers_values_3 { + std::make_tuple(0, "127.0.0.1", 13306, 12, "ONLINE", 1, 1, 1000, 300, 1, 200, "mysql_1"), + std::make_tuple(1, "127.0.0.1", 13307, 13, "OFFLINE_HARD", 2, 1, 500, 300, 1, 200, "mysql_2_offline"), + std::make_tuple(2, "127.0.0.1", 13308, 14, "OFFLINE_HARD", 2, 1, 500, 300, 1, 200, "mysql_3_offline"), + std::make_tuple(3, "127.0.0.1", 13309, 15, "OFFLINE_HARD", 1, 0, 500, 300, 1, 200, "mysql_4_offline") + }; + + check_mysql_servers_sync(cl, proxy_admin, r_proxy_admin, insert_mysql_servers_values_3); + } + { std::string print_master_galera_hostgroups = ""; string_format(t_debug_query, print_master_galera_hostgroups, cl.admin_username, cl.admin_password, cl.host, cl.admin_port, "SELECT * FROM runtime_mysql_galera_hostgroups"); diff --git a/test/tap/tests/test_unsupported_queries-t.cpp b/test/tap/tests/test_unsupported_queries-t.cpp index fff49064fc..35acce78ab 100644 --- a/test/tap/tests/test_unsupported_queries-t.cpp +++ b/test/tap/tests/test_unsupported_queries-t.cpp @@ -66,70 +66,6 @@ using query_test_info = // "SET mysql-enable_load_data_local_infile='true'", - -/** - * @brief Extract the current value for a given 'variable_name' from - * ProxySQL current configuration, either MEMORY or RUNTIME. - * @param proxysql_admin An already opened connection to ProxySQL Admin. - * @param variable_name The name of the variable to be retrieved from ProxySQL - * config. - * @param variable_value Reference to string acting as output parameter which - * will content the value of the specified variable. - * @return EXIT_SUCCESS, or one of the following error codes: - * - EINVAL if supplied 'proxysql_admin' is NULL. - * - '-1' in case of ProxySQL returns an 'NULL' row for the query selecting - * the variable 'sqliteserver-read_only'. - * - EXIT_FAILURE in case other operation failed. - */ -int get_variable_value( - MYSQL* proxysql_admin, const std::string& variable_name, - std::string& variable_value, bool runtime=false -) { - if (proxysql_admin == NULL) { - return EINVAL; - } - - int res = EXIT_FAILURE; - - const std::string t_select_var_query { - "SELECT * FROM %sglobal_variables WHERE Variable_name='%s'" - }; - std::string select_var_query {}; - - if (runtime) { - string_format(t_select_var_query, select_var_query, "runtime_", variable_name.c_str()); - } else { - string_format(t_select_var_query, select_var_query, "", variable_name.c_str()); - } - - MYSQL_QUERY(proxysql_admin, select_var_query.c_str()); - - MYSQL_RES* admin_res = mysql_store_result(proxysql_admin); - if (!admin_res) { - diag("'mysql_store_result' at line %d failed: %s", __LINE__, mysql_error(proxysql_admin)); - goto cleanup; - } - - { - MYSQL_ROW row = mysql_fetch_row(admin_res); - if (!row || row[0] == nullptr || row[1] == nullptr) { - diag("'mysql_fetch_row' at line %d returned 'NULL'", __LINE__); - res = -1; - goto cleanup; - } - - // Extract the result - std::string _variable_value { row[1] }; - variable_value = _variable_value; - - res = EXIT_SUCCESS; - } - -cleanup: - - return res; -} - /** * @brief Enable the query based using the information supplied in the * 'query_info' parameter, and verifies that the value of the query has properly From fa887b2dc99ba06bc4fec360029621aa7c4d077c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Wed, 20 Jul 2022 17:41:03 +0200 Subject: [PATCH 06/26] Prevent Cluster sync between nodes with/without 'ldap_auth' plugin loaded --- lib/ProxySQL_Admin.cpp | 24 ++++++++++++++++++++---- lib/ProxySQL_Cluster.cpp | 7 ++++--- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index a06cb24081..af55711611 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -4476,18 +4476,34 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { if (!strncasecmp("SELECT @@version", query_no_space, strlen("SELECT @@version"))) { l_free(query_length,query); char *q=(char *)"SELECT '%s' AS '@@version'"; - query_length=strlen(q)+20+strlen(PROXYSQL_VERSION); + if (GloMyLdapAuth == nullptr) { + query_length=strlen(q)+20+strlen(PROXYSQL_VERSION); + } else { + query_length=strlen(q)+20+strlen(PROXYSQL_VERSION)+strlen(" Enterprise"); + } query=(char *)l_alloc(query_length); - sprintf(query,q,PROXYSQL_VERSION); + if (GloMyLdapAuth == nullptr) { + sprintf(query, q, PROXYSQL_VERSION); + } else { + sprintf(query, q, PROXYSQL_VERSION" Enterprise"); + } goto __run_query; } if (!strncasecmp("SELECT version()", query_no_space, strlen("SELECT version()"))) { l_free(query_length,query); char *q=(char *)"SELECT '%s' AS 'version()'"; - query_length=strlen(q)+20+strlen(PROXYSQL_VERSION); + if (GloMyLdapAuth == nullptr) { + query_length=strlen(q)+20+strlen(PROXYSQL_VERSION); + } else { + query_length=strlen(q)+20+strlen(PROXYSQL_VERSION)+strlen(" Enterprise"); + } query=(char *)l_alloc(query_length); - sprintf(query,q,PROXYSQL_VERSION); + if (GloMyLdapAuth == nullptr) { + sprintf(query, q, PROXYSQL_VERSION); + } else { + sprintf(query, q, PROXYSQL_VERSION" Enterprise"); + } goto __run_query; } diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index 767013cd4f..160ed607b7 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -114,8 +114,9 @@ void * ProxySQL_Cluster_Monitor_thread(void *args) { bool same_version = false; while ((row = mysql_fetch_row(result))) { if (row[0]) { - if (strcmp(row[0], PROXYSQL_VERSION)==0) { - proxy_info("Cluster: clustering with peer %s:%d . Remote version: %s . Self version: %s\n", node->hostname, node->port, row[0], PROXYSQL_VERSION); + const char* PROXYSQL_VERSION_ = GloMyLdapAuth == nullptr ? PROXYSQL_VERSION : PROXYSQL_VERSION" Enterprise"; + if (strcmp(row[0], PROXYSQL_VERSION_)==0) { + proxy_info("Cluster: clustering with peer %s:%d . Remote version: %s . Self version: %s\n", node->hostname, node->port, row[0], PROXYSQL_VERSION_); same_version = true; std::string q = "PROXYSQL CLUSTER_NODE_UUID "; q += GloVars.uuid; @@ -126,7 +127,7 @@ void * ProxySQL_Cluster_Monitor_thread(void *args) { proxy_info("Cluster: sending CLUSTER_NODE_UUID %s to peer %s:%d\n", GloVars.uuid, node->hostname, node->port); rc_query = mysql_query(conn, q.c_str()); } else { - proxy_warning("Cluster: different ProxySQL version with peer %s:%d . Remote: %s . Self: %s\n", node->hostname, node->port, row[0], PROXYSQL_VERSION); + proxy_warning("Cluster: different ProxySQL version with peer %s:%d . Remote: %s . Self: %s\n", node->hostname, node->port, row[0], PROXYSQL_VERSION_); } } } From a03bf7c58d4a53b198e6ab541ed3ea4208d0f926 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Wed, 20 Jul 2022 17:48:17 +0200 Subject: [PATCH 07/26] Add LDAP support for 'mysql_users' Cluster checksum computation --- lib/ProxySQL_Cluster.cpp | 230 +++++++++++++++++++++++---------------- 1 file changed, 136 insertions(+), 94 deletions(-) diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index 160ed607b7..4e5e807bd4 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -1116,16 +1116,71 @@ int fetch_mysql_users_checksum(MYSQL* conn, char* hostname, uint16_t port, strin return res; } -uint64_t get_mysql_users_checksum(MYSQL_RES* resultset) { +uint64_t get_mysql_users_checksum(MYSQL_RES* resultset, MYSQL_RES* ldap_resultset) { uint64_t raw_users_checksum = GloMyAuth->get_runtime_checksum(resultset); if (GloMyLdapAuth) { - raw_users_checksum += GloMyLdapAuth->get_ldap_mapping_runtime_checksum(); + raw_users_checksum += mysql_raw_checksum(ldap_resultset); } return raw_users_checksum; } +void update_mysql_users(MYSQL_RES* result) { + GloAdmin->admindb->execute("DELETE FROM mysql_users"); + char* q = (char *)"INSERT INTO mysql_users (username, password, active, use_ssl, default_hostgroup, default_schema," + " schema_locked, transaction_persistent, fast_forward, backend, frontend, max_connections, attributes, comment)" + " VALUES (?1 , ?2 , ?3 , ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)"; + + sqlite3_stmt *statement1 = NULL; + int rc = GloAdmin->admindb->prepare_v2(q, &statement1); + ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + + while (MYSQL_ROW row = mysql_fetch_row(result)) { + rc=(*proxy_sqlite3_bind_text)(statement1, 1, row[0], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // username + rc=(*proxy_sqlite3_bind_text)(statement1, 2, row[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // password + rc=(*proxy_sqlite3_bind_int64)(statement1, 3, atoll(row[2])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // active + rc=(*proxy_sqlite3_bind_int64)(statement1, 4, atoll(row[3])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // use_ssl + rc=(*proxy_sqlite3_bind_int64)(statement1, 5, atoll(row[4])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // default_hostgroup + rc=(*proxy_sqlite3_bind_text)(statement1, 6, row[5], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // default_schema + rc=(*proxy_sqlite3_bind_int64)(statement1, 7, atoll(row[6])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // schema_locked + rc=(*proxy_sqlite3_bind_int64)(statement1, 8, atoll(row[7])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // transaction_persistent + rc=(*proxy_sqlite3_bind_int64)(statement1, 9, atoll(row[8])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // fast_forward + rc=(*proxy_sqlite3_bind_int64)(statement1, 10, atoll(row[9])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // backend + rc=(*proxy_sqlite3_bind_int64)(statement1, 11, atoll(row[10])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // frontend + rc=(*proxy_sqlite3_bind_int64)(statement1, 12, atoll(row[11])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // max_connection + rc=(*proxy_sqlite3_bind_text)(statement1, 13, row[12], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // attributes + rc=(*proxy_sqlite3_bind_text)(statement1, 14, row[13], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // comment + + SAFE_SQLITE3_STEP2(statement1); + rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + } +} + +void update_ldap_mappings(MYSQL_RES* result) { + GloAdmin->admindb->execute("DELETE FROM mysql_ldap_mapping"); + char* q = const_cast( + "INSERT INTO mysql_ldap_mapping (priority, frontend_entity, backend_entity, comment)" + " VALUES (?1 , ?2 , ?3 , ?4)" + ); + + sqlite3_stmt *statement1 = NULL; + int rc = GloAdmin->admindb->prepare_v2(q, &statement1); + ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + + while (MYSQL_ROW row = mysql_fetch_row(result)) { + rc=(*proxy_sqlite3_bind_int64)(statement1, 1, atoll(row[0])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // priority + rc=(*proxy_sqlite3_bind_text)(statement1, 2, row[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // frontend_entity + rc=(*proxy_sqlite3_bind_text)(statement1, 3, row[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // backend_entity + rc=(*proxy_sqlite3_bind_text)(statement1, 4, row[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // comment + + SAFE_SQLITE3_STEP2(statement1); + rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + } +} + void ProxySQL_Cluster::pull_mysql_users_from_peer(const string& expected_checksum) { char * hostname = NULL; uint16_t port = 0; @@ -1134,10 +1189,8 @@ void ProxySQL_Cluster::pull_mysql_users_from_peer(const string& expected_checksu if (hostname) { char *username = NULL; char *password = NULL; - // bool rc_bool = true; MYSQL *rc_conn; int rc_query; - int rc; MYSQL *conn = mysql_init(NULL); if (conn==NULL) { proxy_error("Unable to run mysql_init()\n"); @@ -1152,121 +1205,110 @@ void ProxySQL_Cluster::pull_mysql_users_from_peer(const string& expected_checksu //mysql_options(conn, MYSQL_OPT_WRITE_TIMEOUT, &timeout); { unsigned char val = 1; mysql_options(conn, MYSQL_OPT_SSL_ENFORCE, &val); } proxy_info("Cluster: Fetching MySQL Users from peer %s:%d started. Expected checksum: %s\n", hostname, port, expected_checksum.c_str()); - rc_conn = mysql_real_connect(conn, hostname, username, password, NULL, port, NULL, 0); - if (rc_conn) { - rc_query = mysql_query(conn, "SELECT username, password, active, use_ssl, default_hostgroup, default_schema, schema_locked, transaction_persistent, fast_forward, backend, frontend, max_connections, attributes, comment FROM runtime_mysql_users"); - if ( rc_query == 0 ) { - MYSQL_RES *result = mysql_store_result(conn); - proxy_info("Cluster: Fetching MySQL Users from peer %s:%d completed\n", hostname, port); - const uint64_t users_raw_checksum = get_mysql_users_checksum(result); - const string computed_checksum { get_checksum_from_hash(users_raw_checksum) }; + rc_conn = mysql_real_connect(conn, hostname, username, password, NULL, port, NULL, 0); + if (rc_conn == nullptr) { + proxy_info("Cluster: Fetching MySQL Users from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); + metrics.p_counter_array[p_cluster_counter::pulled_mysql_users_failure]->Increment(); - if (expected_checksum == computed_checksum) { + if (GloMyLdapAuth) { + proxy_info("Cluster: Fetching LDAP Mappings from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); + metrics.p_counter_array[p_cluster_counter::pulled_mysql_ldap_mapping_failure]->Increment(); + } - GloAdmin->admindb->execute("DELETE FROM mysql_users"); - MYSQL_ROW row; - char *q = (char *)"INSERT INTO mysql_users (username, password, active, use_ssl, default_hostgroup, default_schema, schema_locked, transaction_persistent, fast_forward, backend, frontend, max_connections, attributes, comment) VALUES (?1 , ?2 , ?3 , ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)"; - sqlite3_stmt *statement1 = NULL; - //sqlite3 *mydb3 = GloAdmin->admindb->get_db(); - //rc=(*proxy_sqlite3_prepare_v2)(mydb3, q, -1, &statement1, 0); - rc = GloAdmin->admindb->prepare_v2(q, &statement1); - ASSERT_SQLITE_OK(rc, GloAdmin->admindb); - while ((row = mysql_fetch_row(result))) { - rc=(*proxy_sqlite3_bind_text)(statement1, 1, row[0], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // username - rc=(*proxy_sqlite3_bind_text)(statement1, 2, row[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // password - rc=(*proxy_sqlite3_bind_int64)(statement1, 3, atoll(row[2])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // active - rc=(*proxy_sqlite3_bind_int64)(statement1, 4, atoll(row[3])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // use_ssl - rc=(*proxy_sqlite3_bind_int64)(statement1, 5, atoll(row[4])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // default_hostgroup - rc=(*proxy_sqlite3_bind_text)(statement1, 6, row[5], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // default_schema - rc=(*proxy_sqlite3_bind_int64)(statement1, 7, atoll(row[6])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // schema_locked - rc=(*proxy_sqlite3_bind_int64)(statement1, 8, atoll(row[7])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // transaction_persistent - rc=(*proxy_sqlite3_bind_int64)(statement1, 9, atoll(row[8])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // fast_forward - rc=(*proxy_sqlite3_bind_int64)(statement1, 10, atoll(row[9])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // backend - rc=(*proxy_sqlite3_bind_int64)(statement1, 11, atoll(row[10])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // frontend - rc=(*proxy_sqlite3_bind_int64)(statement1, 12, atoll(row[11])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // max_connection - rc=(*proxy_sqlite3_bind_text)(statement1, 13, row[12], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // attributes - rc=(*proxy_sqlite3_bind_text)(statement1, 14, row[13], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // comment + goto __exit_pull_mysql_users_from_peer; + } - SAFE_SQLITE3_STEP2(statement1); - rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); - rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); - } - mysql_free_result(result); - proxy_info("Cluster: Loading to runtime MySQL Users from peer %s:%d\n", hostname, port); - GloAdmin->init_users(); - if (GloProxyCluster->cluster_mysql_users_save_to_disk == true) { - proxy_info("Cluster: Saving to disk MySQL Users from peer %s:%d\n", hostname, port); - GloAdmin->flush_mysql_users__from_memory_to_disk(); - } else { - proxy_info("Cluster: Saving to disk MySQL Users Rules from peer %s:%d\n", hostname, port); - } - metrics.p_counter_array[p_cluster_counter::pulled_mysql_users_success]->Increment(); + rc_query = mysql_query( + conn, + "SELECT username, password, active, use_ssl, default_hostgroup, default_schema, schema_locked," + " transaction_persistent, fast_forward, backend, frontend, max_connections, attributes, comment FROM runtime_mysql_users" + ); + if (rc_query == 0) { + MYSQL_RES* mysql_users_result = mysql_store_result(conn); + MYSQL_RES* ldap_mapping_result = nullptr; - metrics.p_counter_array[p_cluster_counter::pulled_mysql_users_success]->Increment(); - } else { - proxy_info( - "Cluster: Fetching MySQL Users from peer %s:%d failed: Checksum changed from %s to %s\n", - hostname, port, expected_checksum.c_str(), computed_checksum.c_str() - ); - metrics.p_counter_array[p_cluster_counter::pulled_mysql_users_failure]->Increment(); - } - } else { - proxy_info("Cluster: Fetching MySQL Users from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); - metrics.p_counter_array[p_cluster_counter::pulled_mysql_users_failure]->Increment(); - } + proxy_info("Cluster: Fetching MySQL Users from peer %s:%d completed\n", hostname, port); - // TODO: Fix LDAP support for cluster synchronization if (GloMyLdapAuth) { + proxy_info("Cluster: Fetching LDAP Mappings from peer %s:%d.\n", hostname, port); + rc_query = mysql_query( - conn, - "SELECT priority, frontend_entity, backend_entity, comment FROM runtime_mysql_ldap_mapping" + conn, "SELECT priority, frontend_entity, backend_entity, comment FROM mysql_ldap_mapping ORDER BY priority" ); if (rc_query == 0) { - MYSQL_RES *result = mysql_store_result(conn); - GloAdmin->admindb->execute("DELETE FROM mysql_ldap_mapping"); - MYSQL_ROW row; - char* q = const_cast( - "INSERT INTO mysql_ldap_mapping (priority, frontend_entity, backend_entity, comment)" - " VALUES (?1 , ?2 , ?3 , ?4)" - ); - sqlite3_stmt *statement1 = NULL; - rc = GloAdmin->admindb->prepare_v2(q, &statement1); - ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + ldap_mapping_result = mysql_store_result(conn); + proxy_info("Cluster: Fetching LDAP Mappings from peer %s:%d completed\n", hostname, port); + } else { + proxy_info("Cluster: Fetching LDAP Mappings from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); + metrics.p_counter_array[p_cluster_counter::pulled_mysql_ldap_mapping_failure]->Increment(); + } + } - while ((row = mysql_fetch_row(result))) { - rc=(*proxy_sqlite3_bind_int64)(statement1, 1, atoll(row[0])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // priority - rc=(*proxy_sqlite3_bind_text)(statement1, 2, row[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // frontend_entity - rc=(*proxy_sqlite3_bind_text)(statement1, 3, row[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // backend_entity - rc=(*proxy_sqlite3_bind_text)(statement1, 4, row[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // comment + const uint64_t users_raw_checksum = get_mysql_users_checksum(mysql_users_result, ldap_mapping_result); + const string computed_checksum { get_checksum_from_hash(users_raw_checksum) }; - SAFE_SQLITE3_STEP2(statement1); - rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); - rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); - } + if (expected_checksum == computed_checksum) { + update_mysql_users(mysql_users_result); + mysql_free_result(mysql_users_result); - mysql_free_result(result); - proxy_info("Cluster: Fetching LDAP Mappings from peer %s:%d completed\n", hostname, port); + if (GloMyLdapAuth) { + update_ldap_mappings(ldap_mapping_result); + mysql_free_result(ldap_mapping_result); + } + + proxy_info("Cluster: Loading to runtime MySQL Users from peer %s:%d\n", hostname, port); + if (GloMyLdapAuth) { proxy_info("Cluster: Loading to runtime LDAP Mappings from peer %s:%d\n", hostname, port); - GloAdmin->init_users(); + } - if (GloProxyCluster->cluster_mysql_users_save_to_disk == true) { + GloAdmin->init_users(); + if (GloProxyCluster->cluster_mysql_users_save_to_disk == true) { + proxy_info("Cluster: Saving to disk MySQL Users from peer %s:%d\n", hostname, port); + if (GloMyLdapAuth) { proxy_info("Cluster: Saving to disk LDAP Mappings from peer %s:%d\n", hostname, port); - GloAdmin->flush_mysql_users__from_memory_to_disk(); - } else { - proxy_info("Cluster: Saving to disk LDAP Mappings Rules from peer %s:%d\n", hostname, port); } - metrics.p_counter_array[p_cluster_counter::pulled_mysql_ldap_mapping_success]->Increment(); + GloAdmin->flush_mysql_users__from_memory_to_disk(); } else { - proxy_info("Cluster: Fetching LDAP Mappings from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); + proxy_info("Cluster: NOT saving to disk MySQL Users from peer %s:%d\n", hostname, port); + if (GloMyLdapAuth) { + proxy_info("Cluster: NOT Saving to disk LDAP Mappings from peer %s:%d\n", hostname, port); + } + } + + metrics.p_counter_array[p_cluster_counter::pulled_mysql_users_success]->Increment(); + + if (GloMyLdapAuth) { + metrics.p_counter_array[p_cluster_counter::pulled_mysql_ldap_mapping_success]->Increment(); + } + } else { + if (mysql_users_result) { + mysql_free_result(mysql_users_result); + } + if (ldap_mapping_result) { + mysql_free_result(ldap_mapping_result); + } + + proxy_info( + "Cluster: Fetching MySQL Users from peer %s:%d failed: Checksum changed from %s to %s\n", + hostname, port, expected_checksum.c_str(), computed_checksum.c_str() + ); + metrics.p_counter_array[p_cluster_counter::pulled_mysql_users_failure]->Increment(); + + if (GloMyLdapAuth) { metrics.p_counter_array[p_cluster_counter::pulled_mysql_ldap_mapping_failure]->Increment(); } } } else { proxy_info("Cluster: Fetching MySQL Users from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); - metrics.p_counter_array[p_cluster_counter::pulled_mysql_ldap_mapping_failure]->Increment(); + metrics.p_counter_array[p_cluster_counter::pulled_mysql_users_failure]->Increment(); + + if (GloMyLdapAuth) { + proxy_info("Cluster: Fetching LDAP Mappings from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); + metrics.p_counter_array[p_cluster_counter::pulled_mysql_ldap_mapping_failure]->Increment(); + } } } __exit_pull_mysql_users_from_peer: From aca6ab62797959f790f819334af6c4c67e770016 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Wed, 20 Jul 2022 17:57:03 +0200 Subject: [PATCH 08/26] Fix crash during startup due to 'GloClickHouseServer' not being yet initialized An invalid memory access can take place if 'GenericRefreshStatistics' is called during startup when 'GloClickHouseServer' isn't yet initialized. --- lib/ProxySQL_Admin.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index af55711611..eabcf008f0 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -6944,7 +6944,11 @@ void ProxySQL_Admin::flush_clickhouse_variables___database_to_runtime(SQLite3DB void ProxySQL_Admin::flush_clickhouse_variables___runtime_to_database(SQLite3DB *db, bool replace, bool del, bool onlyifempty, bool runtime) { proxy_debug(PROXY_DEBUG_ADMIN, 4, "Flushing ClickHouse variables. Replace:%d, Delete:%d, Only_If_Empty:%d\n", replace, del, onlyifempty); - if (GloVars.global.clickhouse_server == false) { + if ( + (GloVars.global.clickhouse_server == false) + || + ( GloClickHouseServer == NULL ) + ) { return; } if (onlyifempty) { From 78da12d2894c0c21b1ef47257f15a46ad6dc252e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Wed, 20 Jul 2022 17:59:11 +0200 Subject: [PATCH 09/26] Removed invalid load from disk of 'mysql_ldap_mapping' in 'ProxySQL_Admin::init' --- lib/ProxySQL_Admin.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index eabcf008f0..9ed84123d7 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -10144,9 +10144,6 @@ void ProxySQL_Admin::__insert_or_replace_maintable_select_disktable() { admindb->execute("INSERT OR REPLACE INTO main.clickhouse_users SELECT * FROM disk.clickhouse_users"); } #endif /* PROXYSQLCLICKHOUSE */ - if (GloMyLdapAuth) { - admindb->execute("INSERT OR REPLACE INTO main.mysql_ldap_mapping SELECT * FROM disk.mysql_ldap_mapping"); - } admindb->execute("PRAGMA foreign_keys = ON"); #if defined(TEST_AURORA) || defined(TEST_GALERA) admindb->execute("DELETE FROM mysql_servers WHERE gtid_port > 0"); // temporary disable add GTID checks From 13a7d7fab0b5addcc4fe2d32515142412daccc6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Wed, 20 Jul 2022 19:55:51 +0200 Subject: [PATCH 10/26] Fix test 'test_cluster1' synchronization for 'mysql_servers' --- test/tap/tests/test_cluster1-t.cpp | 119 ++++++++++++++++++++++------- 1 file changed, 91 insertions(+), 28 deletions(-) diff --git a/test/tap/tests/test_cluster1-t.cpp b/test/tap/tests/test_cluster1-t.cpp index ce017e985f..871de5c1d0 100644 --- a/test/tap/tests/test_cluster1-t.cpp +++ b/test/tap/tests/test_cluster1-t.cpp @@ -1,3 +1,42 @@ +/** + * @file test_cluster1-t.cpp + * @brief This test performs configuration changes in an already setup ProxySQL cluster and verifies the correct + * propagation of changes after the changes. + * @details This test assumes that this proxysql instance is part of a 10 nodes cluster there are 4 core nodes + * and 6 satellite nodes: + * - 127.0.0.1:6032 : this proxy (core node 0) + * - 127.0.0.1:26001 : core node1 + * - 127.0.0.1:26002 : core node2 + * - 127.0.0.1:26003 : core node3 + * - 127.0.0.1:26004 : satellite node1 + * - 127.0.0.1:26005 : satellite node2 + * - 127.0.0.1:26006 : satellite node3 + * - 127.0.0.1:26007 : satellite node4 + * - 127.0.0.1:26008 : satellite node5 + * - 127.0.0.1:26009 : satellite node6 + * + * The test workflow for all the modules that isn't 'mysql_servers' is the following: + * - Get the checksum of the module + * - Update the module + * - Get the new checksum + * - Wait for all the other core nodes to sync + * + * For 'mysql_servers': + * - Get the checksum of the module. + * - Update the module. + * - Get the new checksum. + * - Perform the following actions in loop until core nodes are sync or timeout: + * - Wait for all the other core nodes to sync. + * - If sync fails re-fetch the new checksum from the update node. + * + * This is required for 'mysql_servers' since 'runtime_mysql_servers' could not match the very first fetched checksum + * due to monitoring actions after the configuration is applied. This should be a transitory state, and the whole + * cluster should eventually sync on the same checksum once the required actions for the monitoring are performed. + * This is why the re-fetch of the new checksum is performed for this module. + * + * @date 2022-07-20 + */ + #include #include #include @@ -10,22 +49,6 @@ #include "command_line.h" #include "utils.h" - -/* - * this test assumes that this proxysql instance is part of a 10 nodes cluster - * there are 4 core nodes and 6 satellite nodes - * 127.0.0.1:6032 : this proxy (core node 0) - * 127.0.0.1:26001 : core node1 - * 127.0.0.1:26002 : core node2 - * 127.0.0.1:26003 : core node3 - * 127.0.0.1:26004 : satellite node1 - * 127.0.0.1:26005 : satellite node2 - * 127.0.0.1:26006 : satellite node3 - * 127.0.0.1:26007 : satellite node4 - * 127.0.0.1:26008 : satellite node5 - * 127.0.0.1:26009 : satellite node6 -*/ - int run_q(MYSQL *mysql, const char *q) { MYSQL_QUERY(mysql,q); return 0; @@ -77,18 +100,28 @@ int dumping_checksums_return_uniq(MYSQL_RES *res, std::set& checksu return checksums.size(); } +int _get_checksum(MYSQL* mysql, const std::string& name, std::string& value) { + std::string query { "SELECT checksum FROM runtime_checksums_values WHERE name='" + name + "'" }; + + if (mysql_query(mysql, query.c_str())) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(mysql)); + return -1 ; + } -int get_checksum(MYSQL *mysql, const std::string& name, std::string& value) { - std::string query = "SELECT checksum FROM runtime_checksums_values WHERE name='" + name + "'"; - MYSQL_QUERY(mysql,query.c_str()); MYSQL_RES * res = mysql_store_result(mysql); int rr = mysql_num_rows(res); MYSQL_ROW row; while ((row = mysql_fetch_row(res))) { value = std::string(row[0]); } - ok(rr == 1 && value.length() > 0, "Checksum for %s = %s" , name.c_str(), value.c_str()); mysql_free_result(res); + + return rr; +} + +int get_checksum(MYSQL *mysql, const std::string& name, std::string& value) { + int rr = _get_checksum(mysql, name, value); + ok(rr == 1 && value.length() > 0, "Checksum for %s = %s" , name.c_str(), value.c_str()); if (rr == 1 && value.length() > 0) return 0; return 1; } @@ -135,6 +168,31 @@ int create_connections(CommandLine& cl) { return 0; } +int mysql_servers_in_sync(MYSQL* mysql, const std::string& _servers_chk, int num_retries, int& f_retries) { + std::string servers_chk { _servers_chk }; + + int retries = 0; + int rc = 0; + + while (retries < num_retries) { + int _tmp = 0; + rc = module_in_sync(mysql, "mysql_servers", servers_chk, 1, _tmp); + + if (rc == 0) { + break; + } else { + _get_checksum(mysql, "mysql_servers", servers_chk); + retries++; + + diag("Fetched new 'mysql_servers' target checksum '%s'", servers_chk.c_str()); + } + } + + f_retries = retries; + + return rc; +} + int trigger_sync_and_check(MYSQL *mysql, std::string modname, const char *update_query, const char *load_query) { int rc; std::string chk1; @@ -144,13 +202,23 @@ int trigger_sync_and_check(MYSQL *mysql, std::string modname, const char *update MYSQL_QUERY(mysql, load_query); get_checksum(mysql, modname, chk2); ok(chk1 != chk2 , "%s checksums. Before: %s , after: %s", modname.c_str(), chk1.c_str(), chk2.c_str()); + int retries = 0; - rc = module_in_sync(mysql, modname, chk2, 30, retries); + if (modname != "mysql_servers") { + rc = module_in_sync(mysql, modname, chk2, 30, retries); + } else { + rc = mysql_servers_in_sync(mysql, chk2, 30, retries); + } + ok (rc == 0, "Module %s %sin sync after %d seconds" , modname.c_str() , rc == 0 ? "" : "NOT " , retries); for (int i = 4 ; i Date: Wed, 27 Jul 2022 11:10:14 +0200 Subject: [PATCH 11/26] Perform atomic update of Cluster fetched module checksum epochs --- include/MySQL_HostGroups_Manager.h | 2 +- include/ProxySQL_Cluster.hpp | 10 +-- include/proxysql_admin.h | 46 +++++++++--- lib/MySQL_HostGroups_Manager.cpp | 8 +- lib/ProxySQL_Admin.cpp | 117 +++++++++++++++++++++++------ lib/ProxySQL_Cluster.cpp | 84 ++++++++------------- 6 files changed, 171 insertions(+), 96 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 5d330324a0..db14110cb5 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -538,7 +538,7 @@ class MySQL_HostGroups_Manager { void wrlock(); void wrunlock(); int servers_add(SQLite3_result *resultset); - bool commit(); + bool commit(const std::string& checksum = "", const time_t epoch = 0); void set_incoming_replication_hostgroups(SQLite3_result *); void set_incoming_group_replication_hostgroups(SQLite3_result *); diff --git a/include/ProxySQL_Cluster.hpp b/include/ProxySQL_Cluster.hpp index 6f25bd51b4..035629dba6 100644 --- a/include/ProxySQL_Cluster.hpp +++ b/include/ProxySQL_Cluster.hpp @@ -385,9 +385,9 @@ class ProxySQL_Cluster { void p_update_metrics(); void thread_ending(pthread_t); void join_term_thread(); - void pull_mysql_query_rules_from_peer(const std::string& expected_checksum); - void pull_mysql_servers_from_peer(); - void pull_mysql_users_from_peer(const std::string& expected_checksum); + void pull_mysql_query_rules_from_peer(const std::string& expected_checksum, const time_t epoch); + void pull_mysql_servers_from_peer(const std::string& expected_checksum, const time_t epoch); + void pull_mysql_users_from_peer(const std::string& expected_checksum, const time_t epoch); /** * @brief Pulls from peer the specified global variables by the type parameter. * @param type A string specifying the type of global variables to pull from the peer, supported @@ -395,7 +395,7 @@ class ProxySQL_Cluster { * - 'mysql'. * - 'admin'. */ - void pull_global_variables_from_peer(const std::string& type, const std::string& expected_checksum); - void pull_proxysql_servers_from_peer(const char *expected_checksum); + void pull_global_variables_from_peer(const std::string& type, const std::string& expected_checksum, const time_t epoch); + void pull_proxysql_servers_from_peer(const std::string& expected_checksum, const time_t epoch); }; #endif /* CLASS_PROXYSQL_CLUSTER_H */ diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index 5860bdb39e..624d38de7c 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -235,15 +235,15 @@ class ProxySQL_Admin { void __add_active_users(enum cred_username_type usertype, char *user=NULL, uint64_t *hash1 = NULL); void __delete_inactive_users(enum cred_username_type usertype); void add_admin_users(); - void __refresh_users(); + void __refresh_users(const std::string& checksum = "", const time_t epoch = 0); void __add_active_users_ldap(); void flush_mysql_variables___runtime_to_database(SQLite3DB *db, bool replace, bool del, bool onlyifempty, bool runtime=false, bool use_lock=true); - void flush_mysql_variables___database_to_runtime(SQLite3DB *db, bool replace); + void flush_mysql_variables___database_to_runtime(SQLite3DB *db, bool replace, const std::string& checksum = "", const time_t epoch = 0); char **get_variables_list(); bool set_variable(char *name, char *value); - void flush_admin_variables___database_to_runtime(SQLite3DB *db, bool replace); + void flush_admin_variables___database_to_runtime(SQLite3DB *db, bool replace, const std::string& checksum = "", const time_t epoch = 0); void flush_admin_variables___runtime_to_database(SQLite3DB *db, bool replace, bool del, bool onlyifempty, bool runtime=false); void disk_upgrade_mysql_query_rules(); void disk_upgrade_mysql_servers(); @@ -273,7 +273,7 @@ class ProxySQL_Admin { // LDAP void flush_ldap_variables___runtime_to_database(SQLite3DB *db, bool replace, bool del, bool onlyifempty, bool runtime=false); - void flush_ldap_variables___database_to_runtime(SQLite3DB *db, bool replace); + void flush_ldap_variables___database_to_runtime(SQLite3DB *db, bool replace, const std::string& checksum = "", const time_t epoch = 0); public: pthread_mutex_t sql_query_global_mutex; @@ -306,7 +306,7 @@ class ProxySQL_Admin { bool get_read_only() { return variables.admin_read_only; } bool set_read_only(bool ro) { variables.admin_read_only=ro; return variables.admin_read_only; } bool has_variable(const char *name); - void init_users(); + void init_users(const std::string& checksum = "", const time_t epoch = 0); void init_mysql_servers(); void init_mysql_query_rules(); void init_mysql_firewall(); @@ -339,9 +339,33 @@ class ProxySQL_Admin { // void flush_admin_variables__from_disk_to_memory(); // commented in 2.3 because unused void flush_admin_variables__from_memory_to_disk(); void flush_ldap_variables__from_memory_to_disk(); - void load_mysql_servers_to_runtime(); + void load_mysql_servers_to_runtime(const std::string& checksum = "", const time_t epoch = 0); void save_mysql_servers_from_runtime(); - char * load_mysql_query_rules_to_runtime(SQLite3_result *SQLite3_query_rules_resultset=NULL, SQLite3_result *SQLite3_query_rules_fast_routing_resultset=NULL); + /** + * @brief Performs the load to runtime of the current configuration in 'main' for 'mysql_query_rules' and + * 'mysql_query_rules_fast_routing' and computes the 'mysql_query_rules' module checksum. + * + * @param SQLite3_query_rules_resultset If this parameter is provided, current rows on + * 'mysql_query_rules' are not queried, instead, the contents of the resultset are used. Must + * be the outcome of query 'CLUSTER_QUERY_MYSQL_QUERY_RULES', it's UNSAFE to supply other + * resultset to the function. + * @param SQLite3_query_rules_fast_routing_resultset If this parameter is provided, current rows on + * 'mysql_query_rules_fast_routing' are not queried, instead, the contents of the resultset are used. Must + * be the outcome of query 'CLUSTER_QUERY_MYSQL_QUERY_RULES_FAST_ROUTING', it's UNSAFE to supply other + * resultset to the function. + * @param checksum When used, this parameter must match several requirements depending on the other + * supplied parameters: + * - If the previous two resultset parameters are supplied to this function, this parameter MUST BE the + * already computed checksum from both resultsets combined. + * - When used in combination with the epoch parameter, if the checksum computed for the values from + * tables 'mysql_query_rules' and 'mysql_query_rules_fast_routing' matches this supplied checksum, the + * epoch of 'GloVars.checksums_values.mysql_query_rules.epoch' is updated to be the supplied epoch. + * @param epoch When 'checksum' parameter is supplied, this is the epoch to which the computed checksum + * is to be updated if it matches 'checksum' parameter. + * + * @return Error message in case of not being able to perform the operation, 'NULL' otherwise. + */ + char* load_mysql_query_rules_to_runtime(SQLite3_result* SQLite3_query_rules_resultset=NULL, SQLite3_result* SQLite3_query_rules_fast_routing_resultset=NULL, const std::string& checksum = "", const time_t epoch = 0); void save_mysql_query_rules_from_runtime(bool); void save_mysql_query_rules_fast_routing_from_runtime(bool); char * load_mysql_firewall_to_runtime(); @@ -355,12 +379,12 @@ class ProxySQL_Admin { void flush_scheduler__from_memory_to_disk(); void flush_scheduler__from_disk_to_memory(); - void load_admin_variables_to_runtime() { flush_admin_variables___database_to_runtime(admindb, true); } + void load_admin_variables_to_runtime(const std::string& checksum = "", const time_t epoch = 0) { flush_admin_variables___database_to_runtime(admindb, true, checksum, epoch); } void save_admin_variables_from_runtime() { flush_admin_variables___runtime_to_database(admindb, true, true, false); } void load_or_update_global_settings(SQLite3DB *); - void load_mysql_variables_to_runtime() { flush_mysql_variables___database_to_runtime(admindb, true); } + void load_mysql_variables_to_runtime(const std::string& checksum = "", const time_t epoch = 0) { flush_mysql_variables___database_to_runtime(admindb, true, checksum, epoch); } void save_mysql_variables_from_runtime() { flush_mysql_variables___runtime_to_database(admindb, true, true, false); } void p_update_metrics(); @@ -406,7 +430,7 @@ class ProxySQL_Admin { void flush_configdb(); // 923 // Cluster - void load_proxysql_servers_to_runtime(bool _lock=true); + void load_proxysql_servers_to_runtime(bool _lock=true, const std::string& checksum = "", const time_t epoch = 0); void flush_proxysql_servers__from_memory_to_disk(); void flush_proxysql_servers__from_disk_to_memory(); void save_proxysql_servers_runtime_to_database(bool); @@ -414,7 +438,7 @@ class ProxySQL_Admin { // LDAP void init_ldap_variables(); - void load_ldap_variables_to_runtime() { flush_ldap_variables___database_to_runtime(admindb, true); } + void load_ldap_variables_to_runtime(const std::string& checksum = "", const time_t epoch = 0) { flush_ldap_variables___database_to_runtime(admindb, true, checksum, epoch); } void save_ldap_variables_from_runtime() { flush_ldap_variables___runtime_to_database(admindb, true, true, false); } void save_mysql_ldap_mapping_runtime_to_database(bool); diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index fdd43d2d6e..fd0170e2cb 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -1644,7 +1644,7 @@ SQLite3_result * MySQL_HostGroups_Manager::execute_query(char *query, char **err return resultset; } -bool MySQL_HostGroups_Manager::commit() { +bool MySQL_HostGroups_Manager::commit(const std::string& checksum, const time_t epoch) { unsigned long long curtime1=monotonic_time(); wrlock(); @@ -2051,7 +2051,11 @@ bool MySQL_HostGroups_Manager::commit() { //struct timespec ts; //clock_gettime(CLOCK_REALTIME, &ts); time_t t = time(NULL); - GloVars.checksums_values.mysql_servers.epoch = t; + if (epoch != 0 && checksum != "" && GloVars.checksums_values.mysql_servers.checksum == checksum) { + GloVars.checksums_values.mysql_servers.epoch = epoch; + } else { + GloVars.checksums_values.mysql_servers.epoch = t; + } GloVars.checksums_values.updates_cnt++; GloVars.generate_global_checksum(); GloVars.epoch_version = t; diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 3696d259f6..a64b22ce94 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -6397,7 +6397,7 @@ void ProxySQL_Admin::load_or_update_global_settings(SQLite3DB *db) { } } -void ProxySQL_Admin::flush_admin_variables___database_to_runtime(SQLite3DB *db, bool replace) { +void ProxySQL_Admin::flush_admin_variables___database_to_runtime(SQLite3DB *db, bool replace, const string& checksum, const time_t epoch) { proxy_debug(PROXY_DEBUG_ADMIN, 4, "Flushing ADMIN variables. Replace:%d\n", replace); char *error=NULL; int cols=0; @@ -6464,11 +6464,19 @@ void ProxySQL_Admin::flush_admin_variables___database_to_runtime(SQLite3DB *db, GloVars.checksums_values.admin_variables.set_checksum(buf); GloVars.checksums_values.admin_variables.version++; time_t t = time(NULL); - GloVars.checksums_values.admin_variables.epoch = t; + if (epoch != 0 && checksum != "" && GloVars.checksums_values.admin_variables.checksum == checksum) { + GloVars.checksums_values.admin_variables.epoch = epoch; + } else { + GloVars.checksums_values.admin_variables.epoch = t; + } GloVars.epoch_version = t; GloVars.generate_global_checksum(); GloVars.checksums_values.updates_cnt++; pthread_mutex_unlock(&GloVars.checksum_mutex); + proxy_info( + "Computed checksum for 'LOAD ADMIN VARIABLES TO RUNTIME' was '%s', with epoch '%d'\n", + GloVars.checksums_values.admin_variables.checksum, GloVars.checksums_values.admin_variables.epoch + ); delete resultset; } wrunlock(); @@ -6631,7 +6639,7 @@ void ProxySQL_Admin::flush_admin_variables___database_to_runtime(SQLite3DB *db, if (resultset) delete resultset; } -void ProxySQL_Admin::flush_mysql_variables___database_to_runtime(SQLite3DB *db, bool replace) { +void ProxySQL_Admin::flush_mysql_variables___database_to_runtime(SQLite3DB *db, bool replace, const std::string& checksum, const time_t epoch) { proxy_debug(PROXY_DEBUG_ADMIN, 4, "Flushing MySQL variables. Replace:%d\n", replace); char *error=NULL; int cols=0; @@ -6805,13 +6813,21 @@ void ProxySQL_Admin::flush_mysql_variables___database_to_runtime(SQLite3DB *db, GloVars.checksums_values.mysql_variables.set_checksum(buf); GloVars.checksums_values.mysql_variables.version++; time_t t = time(NULL); - GloVars.checksums_values.mysql_variables.epoch = t; + if (epoch != 0 && checksum != "" && GloVars.checksums_values.mysql_variables.checksum == checksum) { + GloVars.checksums_values.mysql_variables.epoch = epoch; + } else { + GloVars.checksums_values.mysql_variables.epoch = t; + } GloVars.epoch_version = t; GloVars.generate_global_checksum(); GloVars.checksums_values.updates_cnt++; pthread_mutex_unlock(&GloVars.checksum_mutex); delete resultset; } + proxy_info( + "Computed checksum for 'LOAD MYSQL VARIABLES TO RUNTIME' was '%s', with epoch '%d'\n", + GloVars.checksums_values.mysql_variables.checksum, GloVars.checksums_values.mysql_variables.epoch + ); } if (resultset) delete resultset; } @@ -7392,7 +7408,7 @@ void ProxySQL_Admin::flush_mysql_variables___runtime_to_database(SQLite3DB *db, free(varnames); } -void ProxySQL_Admin::flush_ldap_variables___database_to_runtime(SQLite3DB *db, bool replace) { +void ProxySQL_Admin::flush_ldap_variables___database_to_runtime(SQLite3DB *db, bool replace, const std::string& checksum, const time_t epoch) { proxy_debug(PROXY_DEBUG_ADMIN, 4, "Flushing LDAP variables. Replace:%d\n", replace); if (GloMyLdapAuth == NULL) { return; @@ -7460,13 +7476,21 @@ void ProxySQL_Admin::flush_ldap_variables___database_to_runtime(SQLite3DB *db, b GloVars.checksums_values.ldap_variables.set_checksum(buf); GloVars.checksums_values.ldap_variables.version++; time_t t = time(NULL); - GloVars.checksums_values.ldap_variables.epoch = t; + if (epoch != 0 && checksum != "" && GloVars.checksums_values.ldap_variables.checksum == checksum) { + GloVars.checksums_values.ldap_variables.epoch = epoch; + } else { + GloVars.checksums_values.ldap_variables.epoch = t; + } GloVars.epoch_version = t; GloVars.generate_global_checksum(); GloVars.checksums_values.updates_cnt++; pthread_mutex_unlock(&GloVars.checksum_mutex); delete resultset; } + proxy_info( + "Computed checksum for 'LOAD LDAP VARIABLES TO RUNTIME' was '%s', with epoch '%d'\n", + GloVars.checksums_values.ldap_variables.checksum, GloVars.checksums_values.ldap_variables.epoch + ); } if (resultset) delete resultset; } @@ -10457,9 +10481,9 @@ void ProxySQL_Admin::__attach_db(SQLite3DB *db1, SQLite3DB *db2, char *alias) { } -void ProxySQL_Admin::init_users() { +void ProxySQL_Admin::init_users(const std::string& checksum, const time_t epoch) { pthread_mutex_lock(&users_mutex); - __refresh_users(); + __refresh_users(checksum, epoch); pthread_mutex_unlock(&users_mutex); } @@ -10499,7 +10523,7 @@ void ProxySQL_Admin::add_admin_users() { #endif /* DEBUG */ } -void ProxySQL_Admin::__refresh_users() { +void ProxySQL_Admin::__refresh_users(const std::string& checksum, const time_t epoch) { bool calculate_checksum = false; if (checksum_variables.checksum_mysql_users) { calculate_checksum = true; @@ -10540,12 +10564,20 @@ void ProxySQL_Admin::__refresh_users() { GloVars.checksums_values.mysql_users.set_checksum(buf); GloVars.checksums_values.mysql_users.version++; time_t t = time(NULL); - GloVars.checksums_values.mysql_users.epoch = t; + if (epoch != 0 && checksum != "" && GloVars.checksums_values.mysql_users.checksum == checksum) { + GloVars.checksums_values.mysql_users.epoch = epoch; + } else { + GloVars.checksums_values.mysql_users.epoch = t; + } GloVars.epoch_version = t; GloVars.generate_global_checksum(); GloVars.checksums_values.updates_cnt++; pthread_mutex_unlock(&GloVars.checksum_mutex); } + proxy_info( + "Computed checksum for 'LOAD MYSQL USERS TO RUNTIME' was '%s', with epoch '%d'\n", + GloVars.checksums_values.mysql_users.checksum, GloVars.checksums_values.mysql_users.epoch + ); } #ifdef PROXYSQLCLICKHOUSE @@ -11684,7 +11716,7 @@ void ProxySQL_Admin::load_scheduler_to_runtime() { resultset=NULL; } -void ProxySQL_Admin::load_mysql_servers_to_runtime() { +void ProxySQL_Admin::load_mysql_servers_to_runtime(const std::string& checksum, const time_t epoch) { // make sure that the caller has called mysql_servers_wrlock() char *error=NULL; int cols=0; @@ -11819,7 +11851,7 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime() { MyHGM->set_incoming_aws_aurora_hostgroups(resultset_aws_aurora); } // commit all the changes - MyHGM->commit(); + MyHGM->commit(checksum, epoch); GloAdmin->save_mysql_servers_runtime_to_database(true); // clean up @@ -11899,7 +11931,7 @@ char * ProxySQL_Admin::load_mysql_firewall_to_runtime() { return NULL; } -char * ProxySQL_Admin::load_mysql_query_rules_to_runtime(SQLite3_result *SQLite3_query_rules_resultset, SQLite3_result *SQLite3_query_rules_fast_routing_resultset) { +char* ProxySQL_Admin::load_mysql_query_rules_to_runtime(SQLite3_result* SQLite3_query_rules_resultset, SQLite3_result* SQLite3_query_rules_fast_routing_resultset, const std::string& checksum, const time_t epoch) { // About the queries used here, see notes about CLUSTER_QUERY_MYSQL_QUERY_RULES and // CLUSTER_QUERY_MYSQL_QUERY_RULES_FAST_ROUTING in ProxySQL_Cluster.hpp char *error=NULL; @@ -11938,21 +11970,52 @@ char * ProxySQL_Admin::load_mysql_query_rules_to_runtime(SQLite3_result *SQLite3 #endif // BENCHMARK_FASTROUTING_LOAD if (checksum_variables.checksum_mysql_query_rules) { pthread_mutex_lock(&GloVars.checksum_mutex); - uint64_t hash1 = resultset->raw_checksum(); - uint64_t hash2 = resultset2->raw_checksum(); - hash1 += hash2; - uint32_t d32[2]; + char* buff = nullptr; char buf[20]; - memcpy(&d32, &hash1, sizeof(hash1)); - sprintf(buf,"0x%0X%0X", d32[0], d32[1]); - GloVars.checksums_values.mysql_query_rules.set_checksum(buf); + + // If both the resultsets are supplied, then the supplied checksum is the already computed one. + if ( + SQLite3_query_rules_resultset == nullptr || + SQLite3_query_rules_fast_routing_resultset == nullptr + ) { + uint64_t hash1 = resultset->raw_checksum(); + uint64_t hash2 = resultset2->raw_checksum(); + hash1 += hash2; + uint32_t d32[2]; + memcpy(&d32, &hash1, sizeof(hash1)); + sprintf(buf,"0x%0X%0X", d32[0], d32[1]); + + buff = buf; + } else { + buff = const_cast(checksum.c_str()); + } + + GloVars.checksums_values.mysql_query_rules.set_checksum(buff); GloVars.checksums_values.mysql_query_rules.version++; time_t t = time(NULL); - GloVars.checksums_values.mysql_query_rules.epoch = t; + + // Since the supplied checksum is the already computed one and both resultset are + // supplied there is no need for comparsion, because we will be comparing it with itself. + bool same_checksum = + SQLite3_query_rules_resultset != nullptr && + SQLite3_query_rules_fast_routing_resultset != nullptr; + bool matching_checksums = + same_checksum || (GloVars.checksums_values.mysql_query_rules.checksum == checksum); + + if (epoch != 0 && checksum != "" && matching_checksums) { + GloVars.checksums_values.mysql_query_rules.epoch = epoch; + } else { + GloVars.checksums_values.mysql_query_rules.epoch = t; + } + GloVars.epoch_version = t; GloVars.generate_global_checksum(); GloVars.checksums_values.updates_cnt++; pthread_mutex_unlock(&GloVars.checksum_mutex); + proxy_info( + "Computed checksum for 'LOAD MYSQL QUERY RULES TO RUNTIME' was '%s', with epoch '%d'\n", + GloVars.checksums_values.mysql_query_rules.checksum, GloVars.checksums_values.mysql_query_rules.epoch + ); } GloQPro->reset_all(false); QP_rule_t * nqpr; @@ -12840,7 +12903,7 @@ unsigned long long ProxySQL_External_Scheduler::run_once() { return next_run; } -void ProxySQL_Admin::load_proxysql_servers_to_runtime(bool _lock) { +void ProxySQL_Admin::load_proxysql_servers_to_runtime(bool _lock, const std::string& checksum, const time_t epoch) { // make sure that the caller has called mysql_servers_wrlock() char *error=NULL; int cols=0; @@ -12863,11 +12926,19 @@ void ProxySQL_Admin::load_proxysql_servers_to_runtime(bool _lock) { GloVars.checksums_values.proxysql_servers.set_checksum(buf); GloVars.checksums_values.proxysql_servers.version++; time_t t = time(NULL); - GloVars.checksums_values.proxysql_servers.epoch = t; + if (epoch != 0 && checksum != "" && GloVars.checksums_values.proxysql_servers.checksum == checksum) { + GloVars.checksums_values.proxysql_servers.epoch = epoch; + } else { + GloVars.checksums_values.proxysql_servers.epoch = t; + } GloVars.epoch_version = t; GloVars.generate_global_checksum(); GloVars.checksums_values.updates_cnt++; pthread_mutex_unlock(&GloVars.checksum_mutex); + proxy_info( + "Computed checksum for 'LOAD PROXYSQL SERVERS TO RUNTIME' was '%s', with epoch '%d'\n", + GloVars.checksums_values.proxysql_servers.checksum, GloVars.checksums_values.proxysql_servers.epoch + ); // } } if (resultset) delete resultset; diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index b707a5c244..97d691a8f4 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -611,11 +611,7 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { ) { if (v->diff_check >= diff_mqr) { proxy_info("Cluster: detected a peer %s:%d with mysql_query_rules version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); - GloProxyCluster->pull_mysql_query_rules_from_peer(v_exp_checksum); - if (strncmp(v->checksum, GloVars.checksums_values.mysql_query_rules.checksum, 20)==0) { - // we copied from the remote server, let's also copy the same epoch - GloVars.checksums_values.mysql_query_rules.epoch = v->epoch; - } + GloProxyCluster->pull_mysql_query_rules_from_peer(v_exp_checksum, v->epoch); } } if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_mqr*10)) == 0)) { @@ -634,6 +630,8 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { unsigned long long own_version = __sync_fetch_and_add(&GloVars.checksums_values.mysql_servers.version,0); unsigned long long own_epoch = __sync_fetch_and_add(&GloVars.checksums_values.mysql_servers.epoch,0); char* own_checksum = __sync_fetch_and_add(&GloVars.checksums_values.mysql_servers.checksum,0); + const std::string v_exp_checksum { v->checksum }; + if (v->version > 1) { if ( (own_version == 1) // we just booted @@ -642,11 +640,7 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { ) { if (v->diff_check >= diff_ms) { proxy_info("Cluster: detected a peer %s:%d with mysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); - GloProxyCluster->pull_mysql_servers_from_peer(); - if (strncmp(v->checksum, GloVars.checksums_values.mysql_servers.checksum, 20)==0) { - // we copied from the remote server, let's also copy the same epoch - GloVars.checksums_values.mysql_servers.epoch = v->epoch; - } + GloProxyCluster->pull_mysql_servers_from_peer(v_exp_checksum, v->epoch); } } if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_ms*10)) == 0)) { @@ -675,11 +669,7 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { ) { if (v->diff_check >= diff_mu) { proxy_info("Cluster: detected a peer %s:%d with mysql_users version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); - GloProxyCluster->pull_mysql_users_from_peer(v_exp_checksum); - if (strncmp(v->checksum, GloVars.checksums_values.mysql_users.checksum, 20)==0) { - // we copied from the remote server, let's also copy the same epoch - GloVars.checksums_values.mysql_users.epoch = v->epoch; - } + GloProxyCluster->pull_mysql_users_from_peer(v_exp_checksum, v->epoch); } } if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_mu*10)) == 0)) { @@ -708,11 +698,7 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { ) { if (v->diff_check >= diff_mv) { proxy_info("Cluster: detected a peer %s:%d with mysql_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); - GloProxyCluster->pull_global_variables_from_peer("mysql", expected_checksum); - if (strncmp(v->checksum, GloVars.checksums_values.mysql_variables.checksum, 20)==0) { - // we copied from the remote server, let's also copy the same epoch - GloVars.checksums_values.mysql_variables.epoch = v->epoch; - } + GloProxyCluster->pull_global_variables_from_peer("mysql", expected_checksum, v->epoch); } } if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_mv*10)) == 0)) { @@ -741,11 +727,7 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { ) { if (v->diff_check >= diff_av) { proxy_info("Cluster: detected a peer %s:%d with admin_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); - GloProxyCluster->pull_global_variables_from_peer("admin", expected_checksum); - if (strncmp(v->checksum, GloVars.checksums_values.admin_variables.checksum, 20)==0) { - // we copied from the remote server, let's also copy the same epoch - GloVars.checksums_values.admin_variables.epoch = v->epoch; - } + GloProxyCluster->pull_global_variables_from_peer("admin", expected_checksum, v->epoch); } } if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_av*10)) == 0)) { @@ -774,11 +756,7 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { ) { if (v->diff_check >= diff_lv) { proxy_info("Cluster: detected a peer %s:%d with ldap_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); - GloProxyCluster->pull_global_variables_from_peer("ldap", expected_checksum); - if (strncmp(v->checksum, GloVars.checksums_values.ldap_variables.checksum, 20)==0) { - // we copied from the remote server, let's also copy the same epoch - GloVars.checksums_values.ldap_variables.epoch = v->epoch; - } + GloProxyCluster->pull_global_variables_from_peer("ldap", expected_checksum, v->epoch); } } if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_lv*10)) == 0)) { @@ -804,7 +782,7 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { unsigned long long v_epoch = v->epoch; unsigned long long v_version = v->version; unsigned int v_diff_check = v->diff_check; - char* v_exp_checksum = strdup(v->checksum); + const string v_exp_checksum { v->checksum }; if ( (own_version == 1) // we just booted @@ -813,20 +791,13 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { ) { if (v->diff_check >= diff_ps) { proxy_info("Cluster: detected a peer %s:%d with proxysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); - // thus we need to copy it now - GloProxyCluster->pull_proxysql_servers_from_peer((const char *)v_exp_checksum); - if (strncmp(v_exp_checksum, GloVars.checksums_values.proxysql_servers.checksum, 20)==0) { - // we copied from the remote server, let's also copy the same epoch - GloVars.checksums_values.proxysql_servers.epoch = v_epoch; - } + GloProxyCluster->pull_proxysql_servers_from_peer(v_exp_checksum, v->epoch); } } if ((v_epoch == own_epoch) && v_diff_check && ((v_diff_check % (diff_ps*10)) == 0)) { - proxy_error("Cluster: detected a peer %s:%d with proxysql_servers version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %llu checks until LOAD MYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v_version, v_epoch, v_diff_check, v_exp_checksum, own_version, own_epoch, own_checksum, (diff_ps*10)); + proxy_error("Cluster: detected a peer %s:%d with proxysql_servers version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %llu checks until LOAD MYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v_version, v_epoch, v_diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_ps*10)); GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_conflict_proxysql_servers_share_epoch]->Increment(); } - - free(v_exp_checksum); } else { if (v->diff_check && (v->diff_check % (diff_ps*10)) == 0) { proxy_warning("Cluster: detected a peer %s:%d with proxysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %llu checks until LOAD PROXYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_ps*10)); @@ -884,7 +855,7 @@ uint64_t mysql_raw_checksum(MYSQL_RES* resultset) { return res_hash; } -void ProxySQL_Cluster::pull_mysql_query_rules_from_peer(const string& expected_checksum) { +void ProxySQL_Cluster::pull_mysql_query_rules_from_peer(const string& expected_checksum, const time_t epoch) { char * hostname = NULL; uint16_t port = 0; pthread_mutex_lock(&GloProxyCluster->update_mysql_query_rules_mutex); @@ -1032,7 +1003,9 @@ void ProxySQL_Cluster::pull_mysql_query_rules_from_peer(const string& expected_c GloAdmin->admindb->execute("COMMIT"); // We release the ownership of the memory for 'SQLite3' resultsets here since now it's no longer // our responsability to free the memory, they should be directly passed to the 'Query Processor' - GloAdmin->load_mysql_query_rules_to_runtime(SQLite3_query_rules_resultset.release(), SQLite3_query_rules_fast_routing_resultset.release()); + GloAdmin->load_mysql_query_rules_to_runtime( + SQLite3_query_rules_resultset.release(), SQLite3_query_rules_fast_routing_resultset.release(), expected_checksum, epoch + ); if (GloProxyCluster->cluster_mysql_query_rules_save_to_disk == true) { proxy_info("Cluster: Saving to disk MySQL Query Rules from peer %s:%d\n", hostname, port); GloAdmin->flush_mysql_query_rules__from_memory_to_disk(); @@ -1196,7 +1169,7 @@ void update_ldap_mappings(MYSQL_RES* result) { } } -void ProxySQL_Cluster::pull_mysql_users_from_peer(const string& expected_checksum) { +void ProxySQL_Cluster::pull_mysql_users_from_peer(const string& expected_checksum, const time_t epoch) { char * hostname = NULL; uint16_t port = 0; pthread_mutex_lock(&GloProxyCluster->update_mysql_users_mutex); @@ -1278,7 +1251,7 @@ void ProxySQL_Cluster::pull_mysql_users_from_peer(const string& expected_checksu proxy_info("Cluster: Loading to runtime LDAP Mappings from peer %s:%d\n", hostname, port); } - GloAdmin->init_users(); + GloAdmin->init_users(expected_checksum, epoch); if (GloProxyCluster->cluster_mysql_users_save_to_disk == true) { proxy_info("Cluster: Saving to disk MySQL Users from peer %s:%d\n", hostname, port); if (GloMyLdapAuth) { @@ -1491,7 +1464,7 @@ uint64_t compute_servers_tables_raw_checksum(const vector& results) return servers_hash; } -void ProxySQL_Cluster::pull_mysql_servers_from_peer() { +void ProxySQL_Cluster::pull_mysql_servers_from_peer(const std::string& checksum, const time_t epoch) { char * hostname = NULL; uint16_t port = 0; char * peer_checksum = NULL; @@ -1783,7 +1756,7 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer() { delete resultset; proxy_info("Cluster: Loading to runtime MySQL Servers from peer %s:%d\n", hostname, port); - GloAdmin->load_mysql_servers_to_runtime(); + GloAdmin->load_mysql_servers_to_runtime(checksum, epoch); if (GloProxyCluster->cluster_mysql_servers_save_to_disk == true) { proxy_info("Cluster: Saving to disk MySQL Servers from peer %s:%d\n", hostname, port); GloAdmin->flush_mysql_servers__from_memory_to_disk(); @@ -1822,7 +1795,7 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer() { pthread_mutex_unlock(&GloProxyCluster->update_mysql_servers_mutex); } -void ProxySQL_Cluster::pull_global_variables_from_peer(const string& var_type, const string& expected_checksum) { +void ProxySQL_Cluster::pull_global_variables_from_peer(const string& var_type, const string& expected_checksum, const time_t epoch) { char * hostname = NULL; uint16_t port = 0; char* vars_type_str = nullptr; @@ -1941,21 +1914,21 @@ void ProxySQL_Cluster::pull_global_variables_from_peer(const string& var_type, c proxy_info("Cluster: Loading to runtime %s Variables from peer %s:%d\n", vars_type_str, hostname, port); if (var_type == "mysql") { - GloAdmin->load_mysql_variables_to_runtime(); + GloAdmin->load_mysql_variables_to_runtime(expected_checksum, epoch); if (GloProxyCluster->cluster_mysql_variables_save_to_disk == true) { proxy_info("Cluster: Saving to disk MySQL Variables from peer %s:%d\n", hostname, port); GloAdmin->flush_mysql_variables__from_memory_to_disk(); } } else if (var_type == "admin") { - GloAdmin->load_admin_variables_to_runtime(); + GloAdmin->load_admin_variables_to_runtime(expected_checksum, epoch); if (GloProxyCluster->cluster_admin_variables_save_to_disk == true) { proxy_info("Cluster: Saving to disk Admin Variables from peer %s:%d\n", hostname, port); GloAdmin->flush_admin_variables__from_memory_to_disk(); } } else if (var_type == "ldap") { - GloAdmin->load_ldap_variables_to_runtime(); + GloAdmin->load_ldap_variables_to_runtime(expected_checksum, epoch); if (GloProxyCluster->cluster_ldap_variables_save_to_disk == true) { proxy_info("Cluster: Saving to disk LDAP Variables from peer %s:%d\n", hostname, port); @@ -1994,7 +1967,7 @@ void ProxySQL_Cluster::pull_global_variables_from_peer(const string& var_type, c pthread_mutex_unlock(&GloProxyCluster->update_mysql_variables_mutex); } -void ProxySQL_Cluster::pull_proxysql_servers_from_peer(const char *expected_checksum) { +void ProxySQL_Cluster::pull_proxysql_servers_from_peer(const std::string& expected_checksum, const time_t epoch) { char * hostname = NULL; uint16_t port = 0; pthread_mutex_lock(&GloProxyCluster->update_proxysql_servers_mutex); @@ -2018,7 +1991,10 @@ void ProxySQL_Cluster::pull_proxysql_servers_from_peer(const char *expected_chec //mysql_options(conn, MYSQL_OPT_READ_TIMEOUT, &timeout_long); //mysql_options(conn, MYSQL_OPT_WRITE_TIMEOUT, &timeout); { unsigned char val = 1; mysql_options(conn, MYSQL_OPT_SSL_ENFORCE, &val); } - proxy_info("Cluster: Fetching ProxySQL Servers from peer %s:%d started. Expected checksum: %s\n", hostname, port, expected_checksum); + proxy_info( + "Cluster: Fetching ProxySQL Servers from peer %s:%d started. Expected checksum: %s\n", + hostname, port, expected_checksum.c_str() + ); rc_conn = mysql_real_connect(conn, hostname, username, password, NULL, port, NULL, 0); if (rc_conn) { rc_query = mysql_query(conn,"SELECT hostname, port, weight, comment FROM runtime_proxysql_servers ORDER BY hostname, port"); @@ -2058,7 +2034,7 @@ void ProxySQL_Cluster::pull_proxysql_servers_from_peer(const char *expected_chec delete resultset; proxy_info("Cluster: Loading to runtime ProxySQL Servers from peer %s:%d\n", hostname, port); - GloAdmin->load_proxysql_servers_to_runtime(false); + GloAdmin->load_proxysql_servers_to_runtime(false, expected_checksum, epoch); if (GloProxyCluster->cluster_proxysql_servers_save_to_disk == true) { proxy_info("Cluster: Saving to disk ProxySQL Servers from peer %s:%d\n", hostname, port); GloAdmin->flush_proxysql_servers__from_memory_to_disk(); @@ -2069,7 +2045,7 @@ void ProxySQL_Cluster::pull_proxysql_servers_from_peer(const char *expected_chec } else { proxy_info( "Cluster: Fetching ProxySQL Servers from peer %s:%d failed: Checksum changed from %s to %s\n", - hostname, port, expected_checksum, computed_cks.c_str() + hostname, port, expected_checksum.c_str(), computed_cks.c_str() ); metrics.p_counter_array[p_cluster_counter::pulled_proxysql_servers_failure]->Increment(); } From f12839acc1244637418654df6fe307ab0c41fddd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Wed, 27 Jul 2022 12:47:08 +0200 Subject: [PATCH 12/26] Revert "Fix test 'test_cluster1' synchronization for 'mysql_servers'" This reverts commit 13a7d7fab0b5addcc4fe2d32515142412daccc6e. --- test/tap/tests/test_cluster1-t.cpp | 119 +++++++---------------------- 1 file changed, 28 insertions(+), 91 deletions(-) diff --git a/test/tap/tests/test_cluster1-t.cpp b/test/tap/tests/test_cluster1-t.cpp index 871de5c1d0..ce017e985f 100644 --- a/test/tap/tests/test_cluster1-t.cpp +++ b/test/tap/tests/test_cluster1-t.cpp @@ -1,42 +1,3 @@ -/** - * @file test_cluster1-t.cpp - * @brief This test performs configuration changes in an already setup ProxySQL cluster and verifies the correct - * propagation of changes after the changes. - * @details This test assumes that this proxysql instance is part of a 10 nodes cluster there are 4 core nodes - * and 6 satellite nodes: - * - 127.0.0.1:6032 : this proxy (core node 0) - * - 127.0.0.1:26001 : core node1 - * - 127.0.0.1:26002 : core node2 - * - 127.0.0.1:26003 : core node3 - * - 127.0.0.1:26004 : satellite node1 - * - 127.0.0.1:26005 : satellite node2 - * - 127.0.0.1:26006 : satellite node3 - * - 127.0.0.1:26007 : satellite node4 - * - 127.0.0.1:26008 : satellite node5 - * - 127.0.0.1:26009 : satellite node6 - * - * The test workflow for all the modules that isn't 'mysql_servers' is the following: - * - Get the checksum of the module - * - Update the module - * - Get the new checksum - * - Wait for all the other core nodes to sync - * - * For 'mysql_servers': - * - Get the checksum of the module. - * - Update the module. - * - Get the new checksum. - * - Perform the following actions in loop until core nodes are sync or timeout: - * - Wait for all the other core nodes to sync. - * - If sync fails re-fetch the new checksum from the update node. - * - * This is required for 'mysql_servers' since 'runtime_mysql_servers' could not match the very first fetched checksum - * due to monitoring actions after the configuration is applied. This should be a transitory state, and the whole - * cluster should eventually sync on the same checksum once the required actions for the monitoring are performed. - * This is why the re-fetch of the new checksum is performed for this module. - * - * @date 2022-07-20 - */ - #include #include #include @@ -49,6 +10,22 @@ #include "command_line.h" #include "utils.h" + +/* + * this test assumes that this proxysql instance is part of a 10 nodes cluster + * there are 4 core nodes and 6 satellite nodes + * 127.0.0.1:6032 : this proxy (core node 0) + * 127.0.0.1:26001 : core node1 + * 127.0.0.1:26002 : core node2 + * 127.0.0.1:26003 : core node3 + * 127.0.0.1:26004 : satellite node1 + * 127.0.0.1:26005 : satellite node2 + * 127.0.0.1:26006 : satellite node3 + * 127.0.0.1:26007 : satellite node4 + * 127.0.0.1:26008 : satellite node5 + * 127.0.0.1:26009 : satellite node6 +*/ + int run_q(MYSQL *mysql, const char *q) { MYSQL_QUERY(mysql,q); return 0; @@ -100,28 +77,18 @@ int dumping_checksums_return_uniq(MYSQL_RES *res, std::set& checksu return checksums.size(); } -int _get_checksum(MYSQL* mysql, const std::string& name, std::string& value) { - std::string query { "SELECT checksum FROM runtime_checksums_values WHERE name='" + name + "'" }; - - if (mysql_query(mysql, query.c_str())) { - fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(mysql)); - return -1 ; - } +int get_checksum(MYSQL *mysql, const std::string& name, std::string& value) { + std::string query = "SELECT checksum FROM runtime_checksums_values WHERE name='" + name + "'"; + MYSQL_QUERY(mysql,query.c_str()); MYSQL_RES * res = mysql_store_result(mysql); int rr = mysql_num_rows(res); MYSQL_ROW row; while ((row = mysql_fetch_row(res))) { value = std::string(row[0]); } - mysql_free_result(res); - - return rr; -} - -int get_checksum(MYSQL *mysql, const std::string& name, std::string& value) { - int rr = _get_checksum(mysql, name, value); ok(rr == 1 && value.length() > 0, "Checksum for %s = %s" , name.c_str(), value.c_str()); + mysql_free_result(res); if (rr == 1 && value.length() > 0) return 0; return 1; } @@ -168,31 +135,6 @@ int create_connections(CommandLine& cl) { return 0; } -int mysql_servers_in_sync(MYSQL* mysql, const std::string& _servers_chk, int num_retries, int& f_retries) { - std::string servers_chk { _servers_chk }; - - int retries = 0; - int rc = 0; - - while (retries < num_retries) { - int _tmp = 0; - rc = module_in_sync(mysql, "mysql_servers", servers_chk, 1, _tmp); - - if (rc == 0) { - break; - } else { - _get_checksum(mysql, "mysql_servers", servers_chk); - retries++; - - diag("Fetched new 'mysql_servers' target checksum '%s'", servers_chk.c_str()); - } - } - - f_retries = retries; - - return rc; -} - int trigger_sync_and_check(MYSQL *mysql, std::string modname, const char *update_query, const char *load_query) { int rc; std::string chk1; @@ -202,23 +144,13 @@ int trigger_sync_and_check(MYSQL *mysql, std::string modname, const char *update MYSQL_QUERY(mysql, load_query); get_checksum(mysql, modname, chk2); ok(chk1 != chk2 , "%s checksums. Before: %s , after: %s", modname.c_str(), chk1.c_str(), chk2.c_str()); - int retries = 0; - if (modname != "mysql_servers") { - rc = module_in_sync(mysql, modname, chk2, 30, retries); - } else { - rc = mysql_servers_in_sync(mysql, chk2, 30, retries); - } - + rc = module_in_sync(mysql, modname, chk2, 30, retries); ok (rc == 0, "Module %s %sin sync after %d seconds" , modname.c_str() , rc == 0 ? "" : "NOT " , retries); for (int i = 4 ; i Date: Sun, 31 Jul 2022 20:57:48 +0200 Subject: [PATCH 13/26] Cluster: Speedup of processing of MySQL Users and MySQL Servers - Cluster now syncs the server tables via 'incoming_*' tables generated during 'load_mysql_servers_to_runtime'. - Cluster now syncs 'mysql_users' table via a resultset generated via '__refresh_users'. --- include/MySQL_Authentication.hpp | 19 +- include/MySQL_HostGroups_Manager.h | 27 ++ include/ProxySQL_Cluster.hpp | 53 ++-- include/proxysql_admin.h | 37 ++- include/proxysql_structs.h | 2 +- lib/MySQL_Authentication.cpp | 76 ++++-- lib/MySQL_HostGroups_Manager.cpp | 33 +++ lib/ProxySQL_Admin.cpp | 416 +++++++++++++++++++++-------- lib/ProxySQL_Cluster.cpp | 131 +++------ 9 files changed, 552 insertions(+), 242 deletions(-) diff --git a/include/MySQL_Authentication.hpp b/include/MySQL_Authentication.hpp index c1d0ff134c..5947a9165f 100644 --- a/include/MySQL_Authentication.hpp +++ b/include/MySQL_Authentication.hpp @@ -55,6 +55,11 @@ typedef struct _creds_group_t { class MySQL_Authentication { private: + /** + * @brief Holds the current value for 'runtime_mysql_users' used by 'ProxySQL_Admin' to reply to + * 'CLUSTER_QUERY_MYSQL_USERS'. + */ + std::unique_ptr mysql_users_resultset { nullptr }; creds_group_t creds_backends; creds_group_t creds_frontends; bool _reset(enum cred_username_type usertype); @@ -84,9 +89,21 @@ class MySQL_Authentication { * schema_locked, transaction_persistent, fast_forward, backend, frontend, max_connections, * attributes, comment FROM runtime_mysql_users"' * The order isn't relevant in the query itself because ordering is performed while processing. + * @param mysql_users A 'unique_ptr' to be filled with the 'frontend' and 'backend' users found in the + * provided resulset. * @return The computed hash for the provided resultset. */ - uint64_t get_runtime_checksum(MYSQL_RES* resultset); + uint64_t get_runtime_checksum(MYSQL_RES* resultset, unique_ptr& mysql_users); + /** + * @brief Takes ownership of the supplied resultset and stores it in 'mysql_users_resultset' field. + * @param users Holds the current value for 'runtime_mysql_users'. + */ + void save_mysql_users(std::unique_ptr&& users); + /** + * @brief Return a pointer to internally managed 'mysql_users_resultset' field. DO NOT FREE. + * @return A pointer to the internally managed 'mysql_users_resultset'. + */ + SQLite3_result* get_current_mysql_users(); }; #endif /* __CLASS_MYSQL_AUTHENTICATION_H */ diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index db14110cb5..5e3e0ee03e 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -375,6 +375,20 @@ class MySQL_HostGroups_Manager { void generate_mysql_replication_hostgroups_table(); Galera_Info *get_galera_node_info(int hostgroup); + /** + * @brief These resultset holds the latest values for 'incoming_*' tables used promoted servers to runtime. + * @details All these resultsets are used by 'Cluster' to fetch and promote the same configuration used in the + * node across the whole cluster. For these, the queries: + * - 'CLUSTER_QUERY_MYSQL_SERVERS' + * - 'CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS' + * - 'CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS' + * - 'CLUSTER_QUERY_MYSQL_GALERA' + * - 'CLUSTER_QUERY_MYSQL_AWS_AURORA' + * Issued by 'Cluster' are intercepted by 'ProxySQL_Admin' and return the content of these resultsets. This is + * possible because 'incoming_*' tables represent the actual applied configuration to the node servers before + * reconfiguration ('commit') is triggered, making this process convergent on the supplied config. + */ + SQLite3_result* incoming_mysql_servers; SQLite3_result *incoming_replication_hostgroups; void generate_mysql_group_replication_hostgroups_table(); @@ -540,10 +554,23 @@ class MySQL_HostGroups_Manager { int servers_add(SQLite3_result *resultset); bool commit(const std::string& checksum = "", const time_t epoch = 0); + /** + * @brief These setters/getter functions store and retrieve the currently hold resultset for the 'incoming_*' + * table set that have been loaded to runtime. Whose hold the config to be propagated by Cluster. + * @param The resulset to be stored replacing the current one. + */ + void set_incoming_mysql_servers(SQLite3_result *); void set_incoming_replication_hostgroups(SQLite3_result *); void set_incoming_group_replication_hostgroups(SQLite3_result *); void set_incoming_galera_hostgroups(SQLite3_result *); void set_incoming_aws_aurora_hostgroups(SQLite3_result *); + + SQLite3_result* get_current_mysql_servers_inner(); + SQLite3_result* get_current_mysql_replication_hostgroups_inner(); + SQLite3_result* get_current_mysql_group_replication_hostgroups_inner(); + SQLite3_result* get_current_mysql_galera_hostgroups(); + SQLite3_result* get_current_mysql_aws_aurora_hostgroups(); + SQLite3_result * execute_query(char *query, char **error); SQLite3_result *dump_table_mysql_servers(); SQLite3_result *dump_table_mysql_replication_hostgroups(); diff --git a/include/ProxySQL_Cluster.hpp b/include/ProxySQL_Cluster.hpp index 035629dba6..c0d3dc9603 100644 --- a/include/ProxySQL_Cluster.hpp +++ b/include/ProxySQL_Cluster.hpp @@ -12,24 +12,45 @@ #define PROXYSQL_NODE_METRICS_LEN 5 /** - * @brief This query is intercepted by 'ProxySQL_Admin' and doesn't represent the actual resultset being received when - * issuing it. Instead it represent the 'intended' resultset that should be received when fetching a cluster peer - * `MYSQL_SERVERS` table. The caller issuing the query is responsable of the extra-processing. - * @details Received resultset corresponds to 'MySQL_HostGroups_Manager::dump_table_proxysql_servers' resultset. + * CLUSTER QUERIES DEFINITION + * ========================== + * + * The following queries are used by 'ProxySQL_Cluster' and intercepted by 'ProxySQL_Admin'. These queries should match + * the queries issued for generating the checksum for each of the target modules, for simpler reasoning, they should + * also represent the actual resultset being received when issuing them, since this resultset is used for computing the + * 'expected checksum' for the fetched config before loading it to runtime. This is done for the following modules: + * - 'runtime_mysql_servers': tables 'mysql_servers', 'mysql_replication_hostgroups', 'mysql_group_replication_hostroups', + * 'mysql_galera_hostgroups', 'mysql_aws_aurora_hostgroups'. + * - 'runtime_mysql_users'. + * - 'runtime_mysql_query_rules'. + * + * IMPORTANT: For further clarify this means that it's important that the actual resultset produced by the intercepted + * query preserve the filtering and ordering expressed in this queries. */ -#define CLUSTER_QUERY_MYSQL_SERVERS "SELECT hostgroup_id, hostname, port, gtid_port, status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM runtime_mysql_servers WHERE status<>'OFFLINE_HARD' ORDER BY hostgroup_id, hostname, port" -/** - * @brief The result of this query is later used for the computation of the checksum of the received 'mysql_servers'. - * Since it differs from the query being issued by 'MySQL_HostGroups_Manager::commit' for the checksum computation, - * it should be revisit whenever the table 'mysql_replication_hostgroups' change. - */ -#define CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS "SELECT writer_hostgroup, reader_hostgroup, comment FROM runtime_mysql_replication_hostgroups ORDER BY writer_hostgroup" -// the following two queries are the same used in ProxySQL_Admin::load_mysql_query_rules_to_runtime() , but on runtime_ tables. -// It is important to note that the queries in ProxySQL_Admin::load_mysql_query_rules_to_runtime() are used to compute the checksum, -// and are the same resultset saved. Therefore it Cluster can retrieve the same resultset saved it can easily compute the checksum before loading. -#define CLUSTER_QUERY_MYSQL_QUERY_RULES "SELECT rule_id, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, cache_timeout, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, ok_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, attributes, comment FROM runtime_mysql_query_rules ORDER BY rule_id" -#define CLUSTER_QUERY_MYSQL_QUERY_RULES_FAST_ROUTING "SELECT username, schemaname, flagIN, destination_hostgroup, comment FROM runtime_mysql_query_rules_fast_routing ORDER BY username, schemaname, flagIN" +/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_servers'. See top comment for details. */ +#define CLUSTER_QUERY_MYSQL_SERVERS "PROXY_SELECT hostgroup_id, hostname, port, gtid_port, status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM runtime_mysql_servers WHERE status<>'OFFLINE_HARD' ORDER BY hostgroup_id, hostname, port" + +/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_replication_hostgroups'. See top comment for details. */ +#define CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS "PROXY_SELECT writer_hostgroup, reader_hostgroup, comment FROM runtime_mysql_replication_hostgroups ORDER BY writer_hostgroup" + +/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_group_replication_hostgroups'. See top comment for details. */ +#define CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS "PROXY_SELECT writer_hostgroup, backup_writer_hostgroup, reader_hostgroup, offline_hostgroup, active, max_writers, writer_is_also_reader, max_transactions_behind, comment FROM runtime_mysql_group_replication_hostgroups ORDER BY writer_hostgroup" + +/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_aws_aurora_hostgroups'. See top comment for details. */ +#define CLUSTER_QUERY_MYSQL_AWS_AURORA "PROXY_SELECT writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, comment FROM runtime_mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup" + +/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_galera_hostgroups'. See top comment for details. */ +#define CLUSTER_QUERY_MYSQL_GALERA "PROXY_SELECT writer_hostgroup, backup_writer_hostgroup, reader_hostgroup, offline_hostgroup, active, max_writers, writer_is_also_reader, max_transactions_behind, comment FROM runtime_mysql_galera_hostgroups ORDER BY writer_hostgroup" + +/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_users'. See top comment for details. */ +#define CLUSTER_QUERY_MYSQL_USERS "PROXY_SELECT username, password, use_ssl, default_hostgroup, default_schema, schema_locked, transaction_persistent, fast_forward, backend, frontend, max_connections, attributes, comment FROM runtime_mysql_users" + +/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_users'. See top comment for details. */ +#define CLUSTER_QUERY_MYSQL_QUERY_RULES "PROXY_SELECT rule_id, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, cache_timeout, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, ok_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, attributes, comment FROM runtime_mysql_query_rules ORDER BY rule_id" + +/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_users'. See top comment for details. */ +#define CLUSTER_QUERY_MYSQL_QUERY_RULES_FAST_ROUTING "PROXY_SELECT username, schemaname, flagIN, destination_hostgroup, comment FROM runtime_mysql_query_rules_fast_routing ORDER BY username, schemaname, flagIN" class ProxySQL_Checksum_Value_2: public ProxySQL_Checksum_Value { public: diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index 624d38de7c..e0bdd1dcc3 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -116,6 +116,14 @@ struct admin_metrics_map_idx { // ProxySQL_Admin shared variables extern int admin__web_verbosity; +struct incoming_servers_t { + SQLite3_result* incoming_mysql_servers = NULL; + SQLite3_result* incoming_replication_hostgroups = NULL; + SQLite3_result* incoming_group_replication_hostgroups = NULL; + SQLite3_result* incoming_galera_hostgroups = NULL; + SQLite3_result* incoming_aurora_hostgroups = NULL; +}; + class ProxySQL_Admin { private: volatile int main_shutdown; @@ -232,10 +240,31 @@ class ProxySQL_Admin { void __insert_or_replace_disktable_select_maintable(); void __attach_db(SQLite3DB *db1, SQLite3DB *db2, char *alias); - void __add_active_users(enum cred_username_type usertype, char *user=NULL, uint64_t *hash1 = NULL); + /** + * @brief Loads to runtime either supplied users via params or users in 'mysql_users' table. + * @details If the 'usertype' and 'user' parameters are supplied, it loads the target user to runtime. If + * 'user' parameter is not supplied, and 'resulset' param is, it loads the users contained in this + * resultset. If both these params are 'nullptr' current contents of 'mysql_users' table are load to + * runtime. Param 'usertype' is ignored when 'resultset' param is supplied. It always return a + * 'SQLite3_result*' with the users that have been loaded to runtime. + * + * NOTE: The returned resultset doesn't contains duplicated rows for the 'frontend'/'backend' users, + * instead, contains a single row for representing both. This is by design, and the checksum computation + * in the received end should take this into account. + * + * @param usertype The target usertype supplied in param 'user' to 'load to runtime'. + * @param user The username of the user to LOAD TO RUNTIME. + * @param resultset If supplied, must contain all the users to be 'load to runtime'. Typically the + * parameter supplied here is the resultset of query 'CLUSTER_QUERY_MYSQL_USERS'. + * + * @return A 'SQLite3_result*' containing all the users that have been 'loaded to runtime'. When + * param 'resultset' is supplied, it will match it's value, otherwise it will be a locally created + * 'SQLite3_result*' that should be freed. + */ + SQLite3_result* __add_active_users(enum cred_username_type usertype, char *user=NULL, SQLite3_result* resultset = nullptr); void __delete_inactive_users(enum cred_username_type usertype); void add_admin_users(); - void __refresh_users(const std::string& checksum = "", const time_t epoch = 0); + void __refresh_users(std::unique_ptr&& all_users = nullptr, const std::string& checksum = "", const time_t epoch = 0); void __add_active_users_ldap(); void flush_mysql_variables___runtime_to_database(SQLite3DB *db, bool replace, bool del, bool onlyifempty, bool runtime=false, bool use_lock=true); @@ -306,7 +335,7 @@ class ProxySQL_Admin { bool get_read_only() { return variables.admin_read_only; } bool set_read_only(bool ro) { variables.admin_read_only=ro; return variables.admin_read_only; } bool has_variable(const char *name); - void init_users(const std::string& checksum = "", const time_t epoch = 0); + void init_users(std::unique_ptr&& mysql_users_resultset = nullptr, const std::string& checksum = "", const time_t epoch = 0); void init_mysql_servers(); void init_mysql_query_rules(); void init_mysql_firewall(); @@ -339,7 +368,7 @@ class ProxySQL_Admin { // void flush_admin_variables__from_disk_to_memory(); // commented in 2.3 because unused void flush_admin_variables__from_memory_to_disk(); void flush_ldap_variables__from_memory_to_disk(); - void load_mysql_servers_to_runtime(const std::string& checksum = "", const time_t epoch = 0); + void load_mysql_servers_to_runtime(const incoming_servers_t& incoming_servers = {}, const std::string& checksum = "", const time_t epoch = 0); void save_mysql_servers_from_runtime(); /** * @brief Performs the load to runtime of the current configuration in 'main' for 'mysql_query_rules' and diff --git a/include/proxysql_structs.h b/include/proxysql_structs.h index dc7f7efd99..9ccdc7098e 100644 --- a/include/proxysql_structs.h +++ b/include/proxysql_structs.h @@ -39,7 +39,7 @@ enum log_event_type { PROXYSQL_COM_STMT_PREPARE }; -enum cred_username_type { USERNAME_BACKEND, USERNAME_FRONTEND }; +enum cred_username_type { USERNAME_BACKEND, USERNAME_FRONTEND, USERNAME_NONE }; #define PROXYSQL_USE_RESULT diff --git a/lib/MySQL_Authentication.cpp b/lib/MySQL_Authentication.cpp index 0eb8d7014f..1a463082c9 100644 --- a/lib/MySQL_Authentication.cpp +++ b/lib/MySQL_Authentication.cpp @@ -641,7 +641,7 @@ uint64_t MySQL_Authentication::get_runtime_checksum() { return hashB+hashF; } -pair extract_accounts_details(MYSQL_RES* resultset) { +pair extract_accounts_details(MYSQL_RES* resultset, unique_ptr& all_users) { if (resultset == nullptr) { return { umap_auth {}, umap_auth {} }; } // The following order is assumed for the resulset received fields: @@ -650,48 +650,76 @@ pair extract_accounts_details(MYSQL_RES* resultset) { umap_auth f_accs_map {}; umap_auth b_accs_map {}; - while (MYSQL_ROW row = mysql_fetch_row(resultset)) { + // Create the SQLite3 resultsets for 'frontend' and 'backend' users + uint32_t num_fields = mysql_num_fields(resultset); + MYSQL_FIELD* fields = mysql_fetch_fields(resultset); + + SQLite3_result* _all_users { new SQLite3_result(num_fields) }; + + for (uint32_t i = 0; i < num_fields; i++) { + _all_users->add_column_definition(SQLITE_TEXT, fields[i].name); + } + + const auto create_account_details = [] (MYSQL_ROW row) -> account_details_t* { account_details_t* acc_details { new account_details_t {} }; acc_details->username = row[0]; acc_details->password = row[1] ? row[1] : const_cast(""); - acc_details->__active = strcmp(row[2], "1") == 0 ? true : false; - acc_details->use_ssl = strcmp(row[3], "1") == 0 ? true : false; - acc_details->default_hostgroup = atoi(row[4]); - acc_details->default_schema = row[5] ? row[5] : const_cast(""); - acc_details->schema_locked = strcmp(row[6], "1") == 0 ? true : false; - acc_details->transaction_persistent = strcmp(row[7], "1") == 0 ? true : false; - acc_details->fast_forward = strcmp(row[8], "1") == 0 ? true : false; - acc_details->__backend = strcmp(row[9], "1") == 0 ? true : false; - acc_details->__frontend = strcmp(row[10], "1") == 0 ? true : false; - acc_details->max_connections = atoi(row[11]); - acc_details->attributes = row[12] ? row[12] : const_cast(""); - acc_details->comment = row[13] ? row[13] : const_cast(""); + acc_details->__active = true; + acc_details->use_ssl = strcmp(row[2], "1") == 0 ? true : false; + acc_details->default_hostgroup = atoi(row[3]); + acc_details->default_schema = row[4] ? row[4] : const_cast(""); + acc_details->schema_locked = strcmp(row[5], "1") == 0 ? true : false; + acc_details->transaction_persistent = strcmp(row[6], "1") == 0 ? true : false; + acc_details->fast_forward = strcmp(row[7], "1") == 0 ? true : false; + acc_details->__backend = strcmp(row[8], "1") == 0 ? true : false; + acc_details->__frontend = strcmp(row[9], "1") == 0 ? true : false; + acc_details->max_connections = atoi(row[10]); + acc_details->attributes = row[11] ? row[11] : const_cast(""); + acc_details->comment = row[12] ? row[12] : const_cast(""); + + return acc_details; + }; + vector pta(static_cast(num_fields)); + while (MYSQL_ROW row = mysql_fetch_row(resultset)) { // compute the 'username' hash for the map uint64_t u_hash = 0, _u_hash2 = 0; SpookyHash myhash {}; myhash.Init(1,2); - - myhash.Update(acc_details->username, strlen(acc_details->username)); + myhash.Update(row[0], strlen(row[0])); myhash.Final(&u_hash, &_u_hash2); - if (acc_details->__backend) { + // is backend + if (strcmp(row[8], "1") == 0) { + account_details_t* acc_details = create_account_details(row); b_accs_map.insert({u_hash, acc_details}); - } else { + } + // is frontend + if (strcmp(row[9], "1") == 0) { + account_details_t* acc_details = create_account_details(row); f_accs_map.insert({u_hash, acc_details}); } + + // Update the contents of the row for the SQLite3 resultset + for (uint32_t i = 0; i < num_fields; i++) { + pta[i] = row[i]; + } + _all_users->add_row(&pta[0]); } mysql_data_seek(resultset, 0); + // Update the supplied 'unique_ptr' with the target resultsets + all_users.reset(_all_users); + return { b_accs_map, f_accs_map }; } -uint64_t MySQL_Authentication::get_runtime_checksum(MYSQL_RES* resultset) { +uint64_t MySQL_Authentication::get_runtime_checksum(MYSQL_RES* resultset, unique_ptr& all_users) { if (resultset == NULL) { return 0; } - pair acc_maps { extract_accounts_details(resultset) }; + pair acc_maps { extract_accounts_details(resultset, all_users) }; uint64_t b_acc_hash = compute_accounts_hash(acc_maps.first); uint64_t f_acc_hash = compute_accounts_hash(acc_maps.second); @@ -705,3 +733,11 @@ uint64_t MySQL_Authentication::get_runtime_checksum(MYSQL_RES* resultset) { return b_acc_hash + f_acc_hash; } + +void MySQL_Authentication::save_mysql_users(unique_ptr&& users) { + this->mysql_users_resultset = std::move(users); +} + +SQLite3_result* MySQL_Authentication::get_current_mysql_users() { + return this->mysql_users_resultset.get(); +} diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index fd0170e2cb..3e919bd9c1 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -1410,6 +1410,7 @@ MySQL_HostGroups_Manager::MySQL_HostGroups_Manager() { mydb->execute(MYHGM_MYSQL_AWS_AURORA_HOSTGROUPS); mydb->execute("CREATE INDEX IF NOT EXISTS idx_mysql_servers_hostname_port ON mysql_servers (hostname,port)"); MyHostGroups=new PtrArray(); + incoming_mysql_servers=NULL; incoming_replication_hostgroups=NULL; incoming_group_replication_hostgroups=NULL; incoming_galera_hostgroups=NULL; @@ -3788,7 +3789,19 @@ int MySQL_HostGroups_Manager::get_multiple_idle_connections(int _hid, unsigned l return num_conn_current; } +void MySQL_HostGroups_Manager::set_incoming_mysql_servers(SQLite3_result *s) { + if (incoming_mysql_servers) { + delete incoming_mysql_servers; + incoming_mysql_servers = nullptr; + } + incoming_mysql_servers=s; +} + void MySQL_HostGroups_Manager::set_incoming_replication_hostgroups(SQLite3_result *s) { + if (incoming_replication_hostgroups) { + delete incoming_replication_hostgroups; + incoming_replication_hostgroups = nullptr; + } incoming_replication_hostgroups=s; } @@ -3816,6 +3829,26 @@ void MySQL_HostGroups_Manager::set_incoming_aws_aurora_hostgroups(SQLite3_result incoming_aws_aurora_hostgroups=s; } +SQLite3_result* MySQL_HostGroups_Manager::get_current_mysql_servers_inner() { + return this->incoming_mysql_servers; +} + +SQLite3_result* MySQL_HostGroups_Manager::get_current_mysql_replication_hostgroups_inner() { + return this->incoming_replication_hostgroups; +} + +SQLite3_result* MySQL_HostGroups_Manager::get_current_mysql_group_replication_hostgroups_inner() { + return this->incoming_group_replication_hostgroups; +} + +SQLite3_result* MySQL_HostGroups_Manager::get_current_mysql_galera_hostgroups() { + return this->incoming_galera_hostgroups; +} + +SQLite3_result* MySQL_HostGroups_Manager::get_current_mysql_aws_aurora_hostgroups() { + return this->incoming_aws_aurora_hostgroups; +} + SQLite3_result * MySQL_HostGroups_Manager::SQL3_Free_Connections() { const int colnum=13; proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 4, "Dumping Free Connections in Pool\n"); diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index a64b22ce94..96e560c045 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -52,6 +52,9 @@ #include +using std::string; +using std::unique_ptr; + #ifdef WITHGCOV extern "C" void __gcov_dump(); extern "C" void __gcov_reset(); @@ -3698,24 +3701,118 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { // handle special queries from Cluster // for bug #1188 , ProxySQL Admin needs to know the exact query if (!strncasecmp(CLUSTER_QUERY_MYSQL_SERVERS, query_no_space, strlen(CLUSTER_QUERY_MYSQL_SERVERS))) { - //ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; if (sess->session_type == PROXYSQL_SESSION_ADMIN) { // no stats - resultset=MyHGM->dump_table_mysql_servers(); - if (resultset) { + GloAdmin->mysql_servers_wrlock(); + resultset = MyHGM->get_current_mysql_servers_inner(); + GloAdmin->mysql_servers_wrunlock(); + + if (resultset == nullptr) { + resultset=MyHGM->dump_table_mysql_servers(); + if (resultset) { + sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); + delete resultset; + run_query=false; + goto __run_query; + } + } else { sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); - delete resultset; run_query=false; goto __run_query; } } } + if (!strncasecmp(CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS, query_no_space, strlen(CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS))) { - //ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - if (sess->session_type == PROXYSQL_SESSION_ADMIN) { // no stats - resultset=MyHGM->dump_table_mysql_replication_hostgroups(); - if (resultset) { + if (sess->session_type == PROXYSQL_SESSION_ADMIN) { + GloAdmin->mysql_servers_wrlock(); + resultset = MyHGM->get_current_mysql_replication_hostgroups_inner(); + GloAdmin->mysql_servers_wrunlock(); + if (resultset == nullptr) { + resultset=MyHGM->dump_table_mysql_replication_hostgroups(); + if (resultset) { + sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); + delete resultset; + run_query=false; + goto __run_query; + } + } else { + sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); + run_query=false; + goto __run_query; + } + } + } + + if (!strncasecmp(CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS, query_no_space, strlen(CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS))) { + if (sess->session_type == PROXYSQL_SESSION_ADMIN) { + GloAdmin->mysql_servers_wrlock(); + resultset = MyHGM->get_current_mysql_group_replication_hostgroups_inner(); + GloAdmin->mysql_servers_wrunlock(); + if (resultset == nullptr) { + resultset=MyHGM->dump_table_mysql_group_replication_hostgroups(); + if (resultset) { + sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); + delete resultset; + run_query=false; + goto __run_query; + } + } else { + sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); + run_query=false; + goto __run_query; + } + } + } + + if (!strncasecmp(CLUSTER_QUERY_MYSQL_GALERA, query_no_space, strlen(CLUSTER_QUERY_MYSQL_GALERA))) { + if (sess->session_type == PROXYSQL_SESSION_ADMIN) { + GloAdmin->mysql_servers_wrlock(); + resultset = MyHGM->get_current_mysql_galera_hostgroups(); + GloAdmin->mysql_servers_wrunlock(); + if (resultset == nullptr) { + resultset=MyHGM->dump_table_mysql_galera_hostgroups(); + if (resultset) { + sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); + delete resultset; + run_query=false; + goto __run_query; + } + } else { + sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); + run_query=false; + goto __run_query; + } + } + } + + if (!strncasecmp(CLUSTER_QUERY_MYSQL_AWS_AURORA, query_no_space, strlen(CLUSTER_QUERY_MYSQL_AWS_AURORA))) { + if (sess->session_type == PROXYSQL_SESSION_ADMIN) { + GloAdmin->mysql_servers_wrlock(); + resultset = MyHGM->get_current_mysql_aws_aurora_hostgroups(); + GloAdmin->mysql_servers_wrunlock(); + if (resultset == nullptr) { + resultset=MyHGM->dump_table_mysql_aws_aurora_hostgroups(); + if (resultset) { + sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); + delete resultset; + run_query=false; + goto __run_query; + } + } else { + sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); + run_query=false; + goto __run_query; + } + } + } + + if (!strncasecmp(CLUSTER_QUERY_MYSQL_USERS, query_no_space, strlen(CLUSTER_QUERY_MYSQL_USERS))) { + if (sess->session_type == PROXYSQL_SESSION_ADMIN) { + pthread_mutex_lock(&users_mutex); + resultset = GloMyAuth->get_current_mysql_users(); + pthread_mutex_unlock(&users_mutex); + if (resultset != nullptr) { sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); - delete resultset; run_query=false; goto __run_query; } @@ -10481,9 +10578,11 @@ void ProxySQL_Admin::__attach_db(SQLite3DB *db1, SQLite3DB *db2, char *alias) { } -void ProxySQL_Admin::init_users(const std::string& checksum, const time_t epoch) { +void ProxySQL_Admin::init_users( + unique_ptr&& mysql_users_resultset, const std::string& checksum, const time_t epoch +) { pthread_mutex_lock(&users_mutex); - __refresh_users(checksum, epoch); + __refresh_users(std::move(mysql_users_resultset), checksum, epoch); pthread_mutex_unlock(&users_mutex); } @@ -10523,55 +10622,73 @@ void ProxySQL_Admin::add_admin_users() { #endif /* DEBUG */ } -void ProxySQL_Admin::__refresh_users(const std::string& checksum, const time_t epoch) { +void ProxySQL_Admin::__refresh_users( + unique_ptr&& mysql_users_resultset, const string& checksum, const time_t epoch +) { bool calculate_checksum = false; + bool no_resultset_supplied = mysql_users_resultset == nullptr; + if (checksum_variables.checksum_mysql_users) { calculate_checksum = true; } if (calculate_checksum) pthread_mutex_lock(&GloVars.checksum_mutex); + __delete_inactive_users(USERNAME_BACKEND); __delete_inactive_users(USERNAME_FRONTEND); GloMyAuth->set_all_inactive(USERNAME_BACKEND); GloMyAuth->set_all_inactive(USERNAME_FRONTEND); add_admin_users(); -// uint64_t hashB, hashF; -// if (calculate_checksum) { -// __add_active_users(USERNAME_BACKEND, NULL, &hashB); -// __add_active_users(USERNAME_FRONTEND, NULL, &hashF); -// } else { - __add_active_users(USERNAME_BACKEND); - __add_active_users(USERNAME_FRONTEND); -// } + SQLite3_result* added_users { __add_active_users(USERNAME_NONE, NULL, mysql_users_resultset.get()) }; + if (mysql_users_resultset == nullptr && added_users != nullptr) { + mysql_users_resultset.reset(added_users); + } + if (GloMyLdapAuth) { __add_active_users_ldap(); } GloMyAuth->remove_inactives(USERNAME_BACKEND); GloMyAuth->remove_inactives(USERNAME_FRONTEND); - uint64_t hash1 = 0; set_variable((char *)"admin_credentials",(char *)""); + if (calculate_checksum) { - hash1 = GloMyAuth->get_runtime_checksum(); - //uint64_t hash1 = hashB + hashF; // overflow allowed - if (GloMyLdapAuth) { - hash1 += GloMyLdapAuth->get_ldap_mapping_runtime_checksum(); + char* buff = nullptr; + char buf[20] = { 0 }; + + if (no_resultset_supplied) { + uint64_t hash1 = GloMyAuth->get_runtime_checksum(); + if (GloMyLdapAuth) { + hash1 += GloMyLdapAuth->get_ldap_mapping_runtime_checksum(); + } + uint32_t d32[2]; + memcpy(&d32, &hash1, sizeof(hash1)); + sprintf(buf,"0x%0X%0X", d32[0], d32[1]); + + buff = buf; + } else { + buff = const_cast(checksum.c_str()); } - uint32_t d32[2]; - char buf[20]; - memcpy(&d32, &hash1, sizeof(hash1)); - sprintf(buf,"0x%0X%0X", d32[0], d32[1]); - GloVars.checksums_values.mysql_users.set_checksum(buf); + + GloVars.checksums_values.mysql_users.set_checksum(buff); GloVars.checksums_values.mysql_users.version++; time_t t = time(NULL); - if (epoch != 0 && checksum != "" && GloVars.checksums_values.mysql_users.checksum == checksum) { + + const bool same_checksum = no_resultset_supplied == false; + const bool matching_checksums = same_checksum || (GloVars.checksums_values.mysql_users.checksum == checksum); + + if (epoch != 0 && checksum != "" && matching_checksums) { GloVars.checksums_values.mysql_users.epoch = epoch; } else { GloVars.checksums_values.mysql_users.epoch = t; } + GloVars.epoch_version = t; GloVars.generate_global_checksum(); GloVars.checksums_values.updates_cnt++; + + // store the new 'added_users' resultset after generating the new checksum + GloMyAuth->save_mysql_users(std::move(mysql_users_resultset)); pthread_mutex_unlock(&GloVars.checksum_mutex); } proxy_info( @@ -10680,64 +10797,49 @@ void ProxySQL_Admin::__add_active_users_ldap() { -#define ADDUSER_STMT_RAW -void ProxySQL_Admin::__add_active_users(enum cred_username_type usertype, char *__user, uint64_t *hash1) { +SQLite3_result* ProxySQL_Admin::__add_active_users( + enum cred_username_type usertype, char *__user, SQLite3_result* mysql_users_resultset +) { char *error=NULL; int cols=0; int affected_rows=0; - bool empty = true; - SpookyHash myhash; - if (hash1) { - myhash.Init(19,3); - } -#ifdef ADDUSER_STMT_RAW - sqlite3_stmt *statement=NULL; -#else + SQLite3_result *resultset=NULL; -#endif char *str=NULL; char *query=NULL; + if (__user==NULL) { - if (hash1) { - str=(char *)"SELECT username,password,use_ssl,default_hostgroup,default_schema,schema_locked,transaction_persistent,fast_forward,max_connections,attributes,comment FROM main.mysql_users WHERE %s=1 AND active=1 AND default_hostgroup>=0 ORDER BY username"; + if (mysql_users_resultset == nullptr) { + str = (char*)"SELECT username,password,use_ssl,default_hostgroup,default_schema,schema_locked,transaction_persistent,fast_forward,backend,frontend,max_connections,attributes,comment FROM main.mysql_users WHERE active=1 AND default_hostgroup>=0"; + admindb->execute_statement(str, &error, &cols, &affected_rows, &resultset); } else { - str=(char *)"SELECT username,password,use_ssl,default_hostgroup,default_schema,schema_locked,transaction_persistent,fast_forward,max_connections,attributes,comment FROM main.mysql_users WHERE %s=1 AND active=1 AND default_hostgroup>=0"; + resultset = mysql_users_resultset; } - query=(char *)malloc(strlen(str)+15); - sprintf(query,str,(usertype==USERNAME_BACKEND ? "backend" : "frontend")); } else { str=(char *)"SELECT username,password,use_ssl,default_hostgroup,default_schema,schema_locked,transaction_persistent,fast_forward,max_connections,attributes,comment FROM main.mysql_users WHERE %s=1 AND active=1 AND default_hostgroup>=0 AND username='%s'"; query=(char *)malloc(strlen(str)+strlen(__user)+15); sprintf(query,str,(usertype==USERNAME_BACKEND ? "backend" : "frontend"),__user); + admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset); } -#ifdef ADDUSER_STMT_RAW - admindb->execute_statement_raw(query, &error , &cols , &affected_rows , &statement); -#else - admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset); -#endif + + SQLite3_result* added_users { nullptr }; + if (error) { proxy_error("Error on %s : %s\n", query, error); } else { -#ifdef ADDUSER_STMT_RAW - int rc; - while ((rc=(*proxy_sqlite3_step)(statement))==SQLITE_ROW) { - SQLite3_row *r=new SQLite3_row(cols); - r->add_fields(statement); - if (hash1) { - empty = false; - for (int i=0; ifields[i]) { - myhash.Update(r->fields[i],r->sizes[i]); - } else { - myhash.Update("",0); - } - } + SQLite3_result* sqlite_result { nullptr }; + + if (mysql_users_resultset == nullptr) { + sqlite_result = new SQLite3_result(resultset->columns); + + for (SQLite3_column* c : resultset->column_definition) { + sqlite_result->add_column_definition(c->datatype, c->name); } + } + -#else for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { SQLite3_row *r=*it; -#endif char *password=NULL; if (variables.hash_passwords) { // We must use hashed password. See issue #676 // Admin needs to hash the password @@ -10766,44 +10868,86 @@ void ProxySQL_Admin::__add_active_users(enum cred_username_type usertype, char * password=(char *)""; } } - GloMyAuth->add( - r->fields[0], // username - password, // before #676, wewere always passing the password. Now it is possible that the password can be hashed - usertype, // backend/frontend - (strcmp(r->fields[2],"1")==0 ? true : false) , // use_ssl - atoi(r->fields[3]), // default_hostgroup - (r->fields[4]==NULL ? (char *)"" : r->fields[4]), //default_schema - (strcmp(r->fields[5],"1")==0 ? true : false) , // schema_locked - (strcmp(r->fields[6],"1")==0 ? true : false) , // transaction_persistent - (strcmp(r->fields[7],"1")==0 ? true : false), // fast_forward - ( atoi(r->fields[8])>0 ? atoi(r->fields[8]) : 0), // max_connections - (r->fields[9] == NULL ? (char *)"" : r->fields[9]), // attributes - (r->fields[10]==NULL ? (char *)"" : r->fields[10]) //comment - ); + + std::vector usertypes {}; + char* max_connections = nullptr; + char* attributes = nullptr; + char* comment = nullptr; + + if (__user != nullptr) { + usertypes.push_back(usertype); + + max_connections = r->fields[8]; + attributes = r->fields[9]; + comment = r->fields[10]; + } else { + if (strcasecmp(r->fields[8], "1") == 0) { + usertypes.push_back(USERNAME_BACKEND); + } + if (strcasecmp(r->fields[9], "1") == 0) { + usertypes.push_back(USERNAME_FRONTEND); + } + + max_connections = r->fields[10]; + attributes = r->fields[11]; + comment = r->fields[12]; + } + + for (const enum cred_username_type usertype : usertypes) { + GloMyAuth->add( + r->fields[0], // username + password, // before #676, wewere always passing the password. Now it is possible that the password can be hashed + usertype, // backend/frontend + (strcmp(r->fields[2],"1")==0 ? true : false) , // use_ssl + atoi(r->fields[3]), // default_hostgroup + (r->fields[4]==NULL ? (char *)"" : r->fields[4]), //default_schema + (strcmp(r->fields[5],"1")==0 ? true : false) , // schema_locked + (strcmp(r->fields[6],"1")==0 ? true : false) , // transaction_persistent + (strcmp(r->fields[7],"1")==0 ? true : false), // fast_forward + ( atoi(max_connections)>0 ? atoi(max_connections) : 0), // max_connections + (attributes == NULL ? (char *)"" : attributes), // attributes + (comment ==NULL ? (char *)"" : comment) //comment + ); + } + + if (sqlite_result != nullptr) { + vector pta(static_cast(resultset->columns)); + for (uint32_t i = 0; i < resultset->columns; i++) { + if (i == 1) { + pta[i] = password; + } else { + if (r->fields[i] != nullptr) { + pta[i] = r->fields[i]; + } else { + pta[i] = const_cast(""); + } + } + } + sqlite_result->add_row(&pta[0]); + } + if (variables.hash_passwords) { free(password); // because we always generate a new string } -#ifdef ADDUSER_STMT_RAW - delete r; -#endif } - } -#ifdef ADDUSER_STMT_RAW - if (statement) { - (*proxy_sqlite3_finalize)(statement); - } - if (hash1) { - uint64_t h1, h2; - myhash.Final(&h1, &h2); - *hash1 = h1; - if (empty) { - *hash1 = 0; + + if (__user == nullptr) { + if (mysql_users_resultset == nullptr) { + added_users = sqlite_result; + } else { + added_users = mysql_users_resultset; + } + } + + // resulset has been locally allocated and must be deleted + if (resultset != mysql_users_resultset) { + delete resultset; } } -#else - if (resultset) delete resultset; -#endif + free(query); + + return added_users; } #ifdef PROXYSQLCLICKHOUSE @@ -11716,26 +11860,56 @@ void ProxySQL_Admin::load_scheduler_to_runtime() { resultset=NULL; } -void ProxySQL_Admin::load_mysql_servers_to_runtime(const std::string& checksum, const time_t epoch) { +void ProxySQL_Admin::load_mysql_servers_to_runtime( + const incoming_servers_t& incoming_servers, const std::string& checksum, const time_t epoch +) { // make sure that the caller has called mysql_servers_wrlock() char *error=NULL; int cols=0; int affected_rows=0; SQLite3_result *resultset=NULL; + SQLite3_result *resultset_servers=NULL; SQLite3_result *resultset_replication=NULL; SQLite3_result *resultset_group_replication=NULL; SQLite3_result *resultset_galera=NULL; SQLite3_result *resultset_aws_aurora=NULL; - char *query=(char *)"SELECT hostgroup_id,hostname,port,gtid_port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms,comment FROM main.mysql_servers"; + + SQLite3_result* incoming_mysql_servers = incoming_servers.incoming_mysql_servers; + SQLite3_result* incoming_replication_hostgroups = incoming_servers.incoming_replication_hostgroups; + SQLite3_result* incoming_group_replication_hostgroups = incoming_servers.incoming_group_replication_hostgroups; + SQLite3_result* incoming_galera_hostgroups = incoming_servers.incoming_galera_hostgroups; + SQLite3_result* incoming_aurora_hostgroups = incoming_servers.incoming_aurora_hostgroups; + + // TODO: Fix ordering here and place mixed ordering in TEST + char *query=(char *)"SELECT hostgroup_id,hostname,port,gtid_port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms,comment FROM main.mysql_servers ORDER BY hostgroup_id, hostname, port"; proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); - admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset); + if (incoming_mysql_servers == nullptr) { + admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset_servers); + } else { + resultset_servers = incoming_mysql_servers; + } //MyHGH->wrlock(); if (error) { proxy_error("Error on %s : %s\n", query, error); } else { - MyHGM->servers_add(resultset); + MyHGM->servers_add(resultset_servers); + size_t init_row_count = resultset_servers->rows_count; + size_t rm_rows_count = 0; + const auto is_offline_server = [&rm_rows_count] (SQLite3_row* row) { + if (strcasecmp(row->fields[4], "OFFLINE_HARD") == 0) { + rm_rows_count += 1; + return true; + } else { + return false; + } + }; + resultset_servers->rows.erase( + std::remove_if(resultset_servers->rows.begin(), resultset_servers->rows.end(), is_offline_server), + resultset_servers->rows.end() + ); + resultset_servers->rows_count = init_row_count - rm_rows_count; + MyHGM->set_incoming_mysql_servers(resultset_servers); } - if (resultset) delete resultset; resultset=NULL; query=(char *)"SELECT a.* FROM mysql_replication_hostgroups a JOIN mysql_replication_hostgroups b ON a.writer_hostgroup=b.reader_hostgroup WHERE b.reader_hostgroup"; @@ -11752,10 +11926,13 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime(const std::string& checksum, if (resultset) delete resultset; resultset=NULL; - query=(char *)"SELECT a.* FROM mysql_replication_hostgroups a LEFT JOIN mysql_replication_hostgroups b ON a.writer_hostgroup=b.reader_hostgroup WHERE b.reader_hostgroup IS NULL"; + query=(char *)"SELECT a.* FROM mysql_replication_hostgroups a LEFT JOIN mysql_replication_hostgroups b ON a.writer_hostgroup=b.reader_hostgroup WHERE b.reader_hostgroup IS NULL ORDER BY writer_hostgroup"; proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); - admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset_replication); - + if (incoming_replication_hostgroups == nullptr) { + admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset_replication); + } else { + resultset_replication = incoming_replication_hostgroups; + } //MyHGH->wrlock(); if (error) { proxy_error("Error on %s : %s\n", query, error); @@ -11783,9 +11960,14 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime(const std::string& checksum, if (resultset) delete resultset; resultset=NULL; - query=(char *)"SELECT a.* FROM mysql_group_replication_hostgroups a LEFT JOIN mysql_group_replication_hostgroups b ON (a.writer_hostgroup=b.reader_hostgroup OR a.writer_hostgroup=b.backup_writer_hostgroup OR a.writer_hostgroup=b.offline_hostgroup) WHERE b.reader_hostgroup IS NULL AND b.backup_writer_hostgroup IS NULL AND b.offline_hostgroup IS NULL"; + query=(char *)"SELECT a.* FROM mysql_group_replication_hostgroups a LEFT JOIN mysql_group_replication_hostgroups b ON (a.writer_hostgroup=b.reader_hostgroup OR a.writer_hostgroup=b.backup_writer_hostgroup OR a.writer_hostgroup=b.offline_hostgroup) WHERE b.reader_hostgroup IS NULL AND b.backup_writer_hostgroup IS NULL AND b.offline_hostgroup IS NULL ORDER BY writer_hostgroup"; proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); - admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset_group_replication); + if (incoming_group_replication_hostgroups == nullptr) { + admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset_group_replication); + } else { + resultset_group_replication = incoming_group_replication_hostgroups; + } + if (error) { proxy_error("Error on %s : %s\n", query, error); } else { @@ -11810,9 +11992,13 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime(const std::string& checksum, if (resultset) delete resultset; resultset=NULL; - query=(char *)"SELECT a.* FROM mysql_galera_hostgroups a LEFT JOIN mysql_galera_hostgroups b ON (a.writer_hostgroup=b.reader_hostgroup OR a.writer_hostgroup=b.backup_writer_hostgroup OR a.writer_hostgroup=b.offline_hostgroup) WHERE b.reader_hostgroup IS NULL AND b.backup_writer_hostgroup IS NULL AND b.offline_hostgroup IS NULL"; + query=(char *)"SELECT a.* FROM mysql_galera_hostgroups a LEFT JOIN mysql_galera_hostgroups b ON (a.writer_hostgroup=b.reader_hostgroup OR a.writer_hostgroup=b.backup_writer_hostgroup OR a.writer_hostgroup=b.offline_hostgroup) WHERE b.reader_hostgroup IS NULL AND b.backup_writer_hostgroup IS NULL AND b.offline_hostgroup IS NULL ORDER BY writer_hostgroup"; proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); - admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset_galera); + if (incoming_galera_hostgroups == nullptr) { + admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset_galera); + } else { + resultset_galera = incoming_galera_hostgroups; + } if (error) { proxy_error("Error on %s : %s\n", query, error); } else { @@ -11838,12 +12024,16 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime(const std::string& checksum, resultset=NULL; //#ifdef TEST_AURORA // temporary enabled only for testing purpose - query=(char *)"SELECT a.* FROM mysql_aws_aurora_hostgroups a LEFT JOIN mysql_aws_aurora_hostgroups b ON (a.writer_hostgroup=b.reader_hostgroup) WHERE b.reader_hostgroup IS NULL"; + query=(char *)"SELECT a.* FROM mysql_aws_aurora_hostgroups a LEFT JOIN mysql_aws_aurora_hostgroups b ON (a.writer_hostgroup=b.reader_hostgroup) WHERE b.reader_hostgroup IS NULL ORDER BY writer_hostgroup"; //#else // query=(char *)"SELECT a.* FROM mysql_aws_aurora_hostgroups a WHERE 1=0"; //#endif proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); - admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset_aws_aurora); + if (incoming_aurora_hostgroups == nullptr) { + admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset_aws_aurora); + } else { + resultset_aws_aurora = incoming_aurora_hostgroups; + } if (error) { proxy_error("Error on %s : %s\n", query, error); } else { diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index 97d691a8f4..5ffa8a6311 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -894,8 +894,8 @@ void ProxySQL_Cluster::pull_mysql_query_rules_from_peer(const string& expected_c result2 = mysql_store_result(conn); proxy_info("Cluster: Fetching MySQL Query Rules from peer %s:%d completed\n", hostname, port); - std::unique_ptr SQLite3_query_rules_resultset = get_SQLite3_resulset(result1); - std::unique_ptr SQLite3_query_rules_fast_routing_resultset = get_SQLite3_resulset(result2); + std::unique_ptr SQLite3_query_rules_resultset { get_SQLite3_resulset(result1) }; + std::unique_ptr SQLite3_query_rules_fast_routing_resultset { get_SQLite3_resulset(result2) }; const uint64_t query_rules_hash = SQLite3_query_rules_resultset->raw_checksum() + SQLite3_query_rules_fast_routing_resultset->raw_checksum(); @@ -1054,58 +1054,10 @@ void ProxySQL_Cluster::pull_mysql_query_rules_from_peer(const string& expected_c pthread_mutex_unlock(&GloProxyCluster->update_mysql_query_rules_mutex); } -int fetch_mysql_users_checksum(MYSQL* conn, char* hostname, uint16_t port, string& checksum) { - const char* CLUSTER_QUERY_RUNTIME_CHECKS = "SELECT checksum FROM runtime_checksums_values WHERE name='mysql_users' LIMIT 1"; - proxy_info("Cluster: Fetching checksum for MySQL Users from peer %s:%d before processing\n", hostname, port); - - int res = -1; - int my_rc = mysql_query(conn, CLUSTER_QUERY_RUNTIME_CHECKS); - if (!my_rc) { - MYSQL_RES* my_res = mysql_store_result(conn); - if (my_res != NULL && mysql_num_rows(my_res) == 1) { - MYSQL_ROW row = mysql_fetch_row(my_res); - - if (row[0] != nullptr) { - checksum = row[0]; - res = 0; - } else { - proxy_error( - "Cluster: Received empty checksum checksum for MySQL Users from peer %s:%d. Please report a bug\n", - hostname, port - ); - } - } else { - if (mysql_errno(conn)) { - proxy_info( - "Cluster: Fetching checksum for MySQL Users from peer %s:%d failed: '%s'\n", - hostname, port, mysql_error(conn) - ); - } else { - uint64_t num_rows = mysql_num_rows(my_res); - if (num_rows != 1) { - proxy_error( - "Cluster: Empty resulset fetching checksum for MySQL Users from peer %s:%d. Please report a bug\n", - hostname, port - ); - } else { - proxy_error( - "Cluster: Invalid row num '%d' fetching checksum for MySQL Users from peer %s:%d. Please report a bug\n", - num_rows, hostname, port - ); - } - } - } - } else { - proxy_info( - "Cluster: Fetching checksum for MySQL Users from peer %s:%d failed: '%s'\n", hostname, port, mysql_error(conn) - ); - } - - return res; -} - -uint64_t get_mysql_users_checksum(MYSQL_RES* resultset, MYSQL_RES* ldap_resultset) { - uint64_t raw_users_checksum = GloMyAuth->get_runtime_checksum(resultset); +uint64_t get_mysql_users_checksum( + MYSQL_RES* resultset, MYSQL_RES* ldap_resultset, unique_ptr& all_users +) { + uint64_t raw_users_checksum = GloMyAuth->get_runtime_checksum(resultset, all_users); if (GloMyLdapAuth) { raw_users_checksum += mysql_raw_checksum(ldap_resultset); @@ -1127,18 +1079,18 @@ void update_mysql_users(MYSQL_RES* result) { while (MYSQL_ROW row = mysql_fetch_row(result)) { rc=(*proxy_sqlite3_bind_text)(statement1, 1, row[0], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // username rc=(*proxy_sqlite3_bind_text)(statement1, 2, row[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // password - rc=(*proxy_sqlite3_bind_int64)(statement1, 3, atoll(row[2])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // active - rc=(*proxy_sqlite3_bind_int64)(statement1, 4, atoll(row[3])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // use_ssl - rc=(*proxy_sqlite3_bind_int64)(statement1, 5, atoll(row[4])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // default_hostgroup - rc=(*proxy_sqlite3_bind_text)(statement1, 6, row[5], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // default_schema - rc=(*proxy_sqlite3_bind_int64)(statement1, 7, atoll(row[6])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // schema_locked - rc=(*proxy_sqlite3_bind_int64)(statement1, 8, atoll(row[7])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // transaction_persistent - rc=(*proxy_sqlite3_bind_int64)(statement1, 9, atoll(row[8])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // fast_forward - rc=(*proxy_sqlite3_bind_int64)(statement1, 10, atoll(row[9])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // backend - rc=(*proxy_sqlite3_bind_int64)(statement1, 11, atoll(row[10])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // frontend - rc=(*proxy_sqlite3_bind_int64)(statement1, 12, atoll(row[11])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // max_connection - rc=(*proxy_sqlite3_bind_text)(statement1, 13, row[12], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // attributes - rc=(*proxy_sqlite3_bind_text)(statement1, 14, row[13], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // comment + rc=(*proxy_sqlite3_bind_int64)(statement1, 3, 1); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // active + rc=(*proxy_sqlite3_bind_int64)(statement1, 4, atoll(row[2])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // use_ssl + rc=(*proxy_sqlite3_bind_int64)(statement1, 5, atoll(row[3])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // default_hostgroup + rc=(*proxy_sqlite3_bind_text)(statement1, 6, row[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // default_schema + rc=(*proxy_sqlite3_bind_int64)(statement1, 7, atoll(row[5])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // schema_locked + rc=(*proxy_sqlite3_bind_int64)(statement1, 8, atoll(row[6])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // transaction_persistent + rc=(*proxy_sqlite3_bind_int64)(statement1, 9, atoll(row[7])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // fast_forward + rc=(*proxy_sqlite3_bind_int64)(statement1, 10, atoll(row[8])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // backend + rc=(*proxy_sqlite3_bind_int64)(statement1, 11, atoll(row[9])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // frontend + rc=(*proxy_sqlite3_bind_int64)(statement1, 12, atoll(row[10])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // max_connection + rc=(*proxy_sqlite3_bind_text)(statement1, 13, row[11], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // attributes + rc=(*proxy_sqlite3_bind_text)(statement1, 14, row[12], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // comment SAFE_SQLITE3_STEP2(statement1); rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); @@ -1207,11 +1159,7 @@ void ProxySQL_Cluster::pull_mysql_users_from_peer(const string& expected_checksu goto __exit_pull_mysql_users_from_peer; } - rc_query = mysql_query( - conn, - "SELECT username, password, active, use_ssl, default_hostgroup, default_schema, schema_locked," - " transaction_persistent, fast_forward, backend, frontend, max_connections, attributes, comment FROM runtime_mysql_users" - ); + rc_query = mysql_query(conn, CLUSTER_QUERY_MYSQL_USERS); if (rc_query == 0) { MYSQL_RES* mysql_users_result = mysql_store_result(conn); MYSQL_RES* ldap_mapping_result = nullptr; @@ -1234,7 +1182,9 @@ void ProxySQL_Cluster::pull_mysql_users_from_peer(const string& expected_checksu } } - const uint64_t users_raw_checksum = get_mysql_users_checksum(mysql_users_result, ldap_mapping_result); + unique_ptr mysql_users_resultset { nullptr }; + const uint64_t users_raw_checksum = + get_mysql_users_checksum(mysql_users_result, ldap_mapping_result, mysql_users_resultset); const string computed_checksum { get_checksum_from_hash(users_raw_checksum) }; if (expected_checksum == computed_checksum) { @@ -1251,7 +1201,7 @@ void ProxySQL_Cluster::pull_mysql_users_from_peer(const string& expected_checksu proxy_info("Cluster: Loading to runtime LDAP Mappings from peer %s:%d\n", hostname, port); } - GloAdmin->init_users(expected_checksum, epoch); + GloAdmin->init_users(std::move(mysql_users_resultset), expected_checksum, epoch); if (GloProxyCluster->cluster_mysql_users_save_to_disk == true) { proxy_info("Cluster: Saving to disk MySQL Users from peer %s:%d\n", hostname, port); if (GloMyLdapAuth) { @@ -1464,6 +1414,20 @@ uint64_t compute_servers_tables_raw_checksum(const vector& results) return servers_hash; } +incoming_servers_t convert_servers_resultsets(const std::vector& results) { + if (results.size() != sizeof(incoming_servers_t) / sizeof(void*)) { + return {}; + } else { + return { + get_SQLite3_resulset(results[0]).release(), + get_SQLite3_resulset(results[1]).release(), + get_SQLite3_resulset(results[2]).release(), + get_SQLite3_resulset(results[3]).release(), + get_SQLite3_resulset(results[4]).release() + }; + } +} + void ProxySQL_Cluster::pull_mysql_servers_from_peer(const std::string& checksum, const time_t epoch) { char * hostname = NULL; uint16_t port = 0; @@ -1493,34 +1457,25 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer(const std::string& checksum, if (rc_conn) { std::vector results {}; - // Server query messages + // servers messages std::string fetch_servers_done = ""; string_format("Cluster: Fetching MySQL Servers from peer %s:%d completed\n", fetch_servers_done, hostname, port); std::string fetch_servers_err = ""; string_format("Cluster: Fetching MySQL Servers from peer %s:%d failed: \n", fetch_servers_err, hostname, port); - // group_replication_hostgroups query and messages - const char* CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS = - "SELECT writer_hostgroup, backup_writer_hostgroup, reader_hostgroup, offline_hostgroup, active, " - "max_writers, writer_is_also_reader, max_transactions_behind, comment FROM runtime_mysql_group_replication_hostgroups ORDER BY writer_hostgroup"; + // group_replication_hostgroups messages std::string fetch_group_replication_hostgroups = ""; string_format("Cluster: Fetching 'MySQL Group Replication Hostgroups' from peer %s:%d\n", fetch_group_replication_hostgroups, hostname, port); std::string fetch_group_replication_hostgroups_err = ""; string_format("Cluster: Fetching 'MySQL Group Replication Hostgroups' from peer %s:%d failed: \n", fetch_group_replication_hostgroups_err, hostname, port); - // AWS Aurora query and messages - const char* CLUSTER_QUERY_MYSQL_AWS_AURORA = - "SELECT writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, " - "check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, comment FROM runtime_mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup"; + // AWS Aurora messages std::string fetch_aws_aurora_start = ""; string_format("Cluster: Fetching 'MySQL Aurora Hostgroups' from peer %s:%d\n", fetch_aws_aurora_start, hostname, port); std::string fetch_aws_aurora_err = ""; string_format("Cluster: Fetching 'MySQL Aurora Hostgroups' from peer %s:%d failed: \n", fetch_aws_aurora_err, hostname, port); - // Galera query and messages - const char* CLUSTER_QUERY_MYSQL_GALERA = - "SELECT writer_hostgroup, backup_writer_hostgroup, reader_hostgroup, offline_hostgroup, active, " - "max_writers, writer_is_also_reader, max_transactions_behind, comment FROM runtime_mysql_galera_hostgroups ORDER BY writer_hostgroup"; + // Galera messages std::string fetch_galera_start = ""; string_format("Cluster: Fetching 'MySQL Galera Hostgroups' from peer %s:%d\n", fetch_galera_start, hostname, port); std::string fetch_galera_err = ""; @@ -1583,6 +1538,8 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer(const std::string& checksum, const string computed_checksum { get_checksum_from_hash(servers_hash) }; if (computed_checksum == peer_checksum) { + // No need to perform the conversion if checksums don't match + const incoming_servers_t incoming_servers { convert_servers_resultsets(results) }; // we are OK to sync! proxy_info("Cluster: Fetching checksum for 'MySQL Servers' from peer %s:%d successful. Checksum: %s\n", hostname, port, computed_checksum.c_str()); // sync mysql_servers @@ -1756,7 +1713,7 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer(const std::string& checksum, delete resultset; proxy_info("Cluster: Loading to runtime MySQL Servers from peer %s:%d\n", hostname, port); - GloAdmin->load_mysql_servers_to_runtime(checksum, epoch); + GloAdmin->load_mysql_servers_to_runtime(incoming_servers, checksum, epoch); if (GloProxyCluster->cluster_mysql_servers_save_to_disk == true) { proxy_info("Cluster: Saving to disk MySQL Servers from peer %s:%d\n", hostname, port); GloAdmin->flush_mysql_servers__from_memory_to_disk(); From a999eeddca6242bc2554e70db9fbf683fc1b5260 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Sun, 31 Jul 2022 21:17:24 +0200 Subject: [PATCH 14/26] Fix 'test_cluster1-t' checksum fetching logic --- test/tap/tests/test_cluster1-t.cpp | 57 +++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 17 deletions(-) diff --git a/test/tap/tests/test_cluster1-t.cpp b/test/tap/tests/test_cluster1-t.cpp index ce017e985f..b3a4526cef 100644 --- a/test/tap/tests/test_cluster1-t.cpp +++ b/test/tap/tests/test_cluster1-t.cpp @@ -77,29 +77,43 @@ int dumping_checksums_return_uniq(MYSQL_RES *res, std::set& checksu return checksums.size(); } +int _get_checksum(MYSQL* mysql, const std::string& name, std::string& value) { + std::string query { "SELECT checksum FROM runtime_checksums_values WHERE name='" + name + "'" }; + + if (mysql_query(mysql, query.c_str())) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(mysql)); + return -1; + } -int get_checksum(MYSQL *mysql, const std::string& name, std::string& value) { - std::string query = "SELECT checksum FROM runtime_checksums_values WHERE name='" + name + "'"; - MYSQL_QUERY(mysql,query.c_str()); MYSQL_RES * res = mysql_store_result(mysql); int rr = mysql_num_rows(res); MYSQL_ROW row; while ((row = mysql_fetch_row(res))) { value = std::string(row[0]); } - ok(rr == 1 && value.length() > 0, "Checksum for %s = %s" , name.c_str(), value.c_str()); mysql_free_result(res); + + return rr; +} + +int get_checksum(MYSQL *mysql, const std::string& name, std::string& value) { + int rr = _get_checksum(mysql, name, value); + ok(rr == 1 && value.length() > 0, "Checksum for %s = %s" , name.c_str(), value.c_str()); if (rr == 1 && value.length() > 0) return 0; return 1; } -int module_in_sync(MYSQL *mysql, const std::string& name, const std::string& value, int num_retries, int& i) { +int module_in_sync( + MYSQL* tg_admin, MYSQL* fetch_conn, const std::string& name, const std::string& init_chk, int num_retries, int& i +) { std::string query = "SELECT hostname, port, name, version, epoch, checksum, changed_at, updated_at, diff_check FROM stats_proxysql_servers_checksums WHERE name='" + name + "'"; + std::string checksum { init_chk }; int rc = 0; + while (i< num_retries && rc != 1) { std::set checksums; - MYSQL_QUERY(mysql,query.c_str()); - MYSQL_RES * res = mysql_store_result(mysql); + MYSQL_QUERY(tg_admin, query.c_str()); + MYSQL_RES * res = mysql_store_result(tg_admin); std::string s; get_time(s); diag("%s: Dumping %s", s.c_str(), query.c_str()); @@ -107,13 +121,21 @@ int module_in_sync(MYSQL *mysql, const std::string& name, const std::string& val mysql_free_result(res); if (rc == 1) { std::set::iterator it = checksums.begin(); - if (*it == value) { + if (*it == checksum) { return 0; + } else { + int chk_res = _get_checksum(fetch_conn, name, checksum); + if (chk_res != -1) { + diag("Fetched new '%s' target checksum '%s'", name.c_str(), checksum.c_str()); + } else { + diag("Fetching new checksum for module '%s' failed", name.c_str()); + } } } sleep(1); i++; } + return 1; } @@ -145,12 +167,12 @@ int trigger_sync_and_check(MYSQL *mysql, std::string modname, const char *update get_checksum(mysql, modname, chk2); ok(chk1 != chk2 , "%s checksums. Before: %s , after: %s", modname.c_str(), chk1.c_str(), chk2.c_str()); int retries = 0; - rc = module_in_sync(mysql, modname, chk2, 30, retries); + rc = module_in_sync(mysql, mysql, modname, chk2, 30, retries); ok (rc == 0, "Module %s %sin sync after %d seconds" , modname.c_str() , rc == 0 ? "" : "NOT " , retries); for (int i = 4 ; i Date: Sun, 31 Jul 2022 21:18:50 +0200 Subject: [PATCH 15/26] Fix minor typos in metrics descriptions --- lib/ProxySQL_Admin.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 96e560c045..52d549eb03 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -777,7 +777,7 @@ admin_metrics_map = std::make_tuple( // TODO: Check why 'global_mysql_firewall_whitelist_users_result___size' never updated p_admin_gauge::mysql_firewall_users_config, "proxysql_mysql_firewall_users_config_bytes", - "Full 'mysql_firewall_users' config 'resulset' size.", + "Full 'mysql_firewall_users' config 'resultset' size.", metric_tags {} ), std::make_tuple ( @@ -789,7 +789,7 @@ admin_metrics_map = std::make_tuple( std::make_tuple ( p_admin_gauge::mysql_firewall_rules_config, "proxysql_mysql_firewall_rules_config_bytes", - "Full 'mysql_firewall_users' config 'resulset' size.", + "Full 'mysql_firewall_users' config 'resultset' size.", metric_tags {} ), std::make_tuple ( From c46a805c706213634da1444083be513fa56761f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Sun, 31 Jul 2022 21:19:07 +0200 Subject: [PATCH 16/26] Fix memory leak for 'Scheduler_Row::filename' field --- lib/ProxySQL_Admin.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 52d549eb03..6a10bac79e 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -12928,6 +12928,9 @@ Scheduler_Row::~Scheduler_Row() { } args[i]=NULL; } + if (filename) { + free(filename); + } free(args); free(comment); args=NULL; From b34100ceec2659e995b0cf2b3e10c6d4790ff18f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Sun, 31 Jul 2022 21:20:27 +0200 Subject: [PATCH 17/26] Fix memory leaks for 'username' and 'password' credentials in cluster 'pull' actions --- lib/ProxySQL_Cluster.cpp | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index 5ffa8a6311..5e1d8bc524 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -1249,6 +1249,12 @@ void ProxySQL_Cluster::pull_mysql_users_from_peer(const string& expected_checksu } } } + if (username) { + free(username); + } + if (password) { + free(password); + } __exit_pull_mysql_users_from_peer: if (conn) { if (conn->net.pvio) { @@ -1741,6 +1747,12 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer(const std::string& checksum, metrics.p_counter_array[p_cluster_counter::pulled_mysql_servers_failure]->Increment(); } } + if (username) { + free(username); + } + if (password) { + free(password); + } __exit_pull_mysql_servers_from_peer: if (conn) { if (conn->net.pvio) { @@ -1913,6 +1925,12 @@ void ProxySQL_Cluster::pull_global_variables_from_peer(const string& var_type, c metrics.p_counter_array[failure_metric]->Increment(); } } + if (username) { + free(username); + } + if (password) { + free(password); + } __exit_pull_mysql_variables_from_peer: if (conn) { if (conn->net.pvio) { @@ -2016,6 +2034,12 @@ void ProxySQL_Cluster::pull_proxysql_servers_from_peer(const std::string& expect metrics.p_counter_array[p_cluster_counter::pulled_proxysql_servers_failure]->Increment(); } } + if (username) { + free(username); + } + if (password) { + free(password); + } __exit_pull_proxysql_servers_from_peer: if (conn) { if (conn->net.pvio) { From bf1d70aa2873c57dbfee3b23a2f4b2c563ca06ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Sun, 31 Jul 2022 21:23:09 +0200 Subject: [PATCH 18/26] Fix memory leaks in 'Query_Processor' for Query Rules 'attributes' and 'comment' fields --- lib/Query_Processor.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/Query_Processor.cpp b/lib/Query_Processor.cpp index 51898fa460..11f0c8f246 100644 --- a/lib/Query_Processor.cpp +++ b/lib/Query_Processor.cpp @@ -386,6 +386,10 @@ static void __delete_query_rule(QP_rule_t *qr) { free(qr->error_msg); if (qr->OK_msg) free(qr->OK_msg); + if (qr->attributes) + free(qr->attributes); + if (qr->comment) + free(qr->comment); if (qr->regex_engine1) { re2_t *r=(re2_t *)qr->regex_engine1; if (r->opt1) { delete r->opt1; r->opt1=NULL; } From 0335bdc5b7d0816821b72551346f8d03eb0133d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Sun, 31 Jul 2022 21:27:40 +0200 Subject: [PATCH 19/26] Change servers entries order in 'test_cluster_sync-t' to test checksum computation --- test/tap/tests/test_cluster_sync-t.cpp | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/test/tap/tests/test_cluster_sync-t.cpp b/test/tap/tests/test_cluster_sync-t.cpp index 807f8b63b5..35c504ab69 100644 --- a/test/tap/tests/test_cluster_sync-t.cpp +++ b/test/tap/tests/test_cluster_sync-t.cpp @@ -517,10 +517,10 @@ int main(int, char**) { { vector insert_mysql_servers_values { - std::make_tuple(0, "127.0.0.1", 13306, 12, "ONLINE", 1, 1, 1000, 300, 1, 200, ""), - std::make_tuple(1, "127.0.0.1", 13307, 13, "OFFLINE_SOFT", 2, 1, 500, 300, 1, 200, ""), std::make_tuple(2, "127.0.0.1", 13308, 14, "OFFLINE_HARD", 2, 1, 500, 300, 1, 200, ""), - std::make_tuple(3, "127.0.0.1", 13309, 15, "SHUNNED", 1, 0, 500, 300, 1, 200, "") + std::make_tuple(3, "127.0.0.1", 13309, 15, "SHUNNED", 1, 0, 500, 300, 1, 200, ""), + std::make_tuple(0, "127.0.0.1", 13306, 12, "ONLINE", 1, 1, 1000, 300, 1, 200, ""), + std::make_tuple(1, "127.0.0.1", 13307, 13, "OFFLINE_SOFT", 2, 1, 500, 300, 1, 200, "") }; check_mysql_servers_sync(cl, proxy_admin, r_proxy_admin, insert_mysql_servers_values); @@ -675,10 +675,10 @@ int main(int, char**) { "active, max_writers, writer_is_also_reader, max_transactions_behind, comment) " "VALUES (%d, %d, %d, %d, %d, %d, %d, %d, %s)"; std::vector> insert_galera_values { + std::make_tuple(3, 7, 11, 15, 1, 20, 0, 350, "'reader_writer_test_galera_hostgroup'"), std::make_tuple(0, 4, 8, 12, 1, 10, 0, 200, "'reader_writer_test_galera_hostgroup'"), - std::make_tuple(1, 5, 9, 13, 1, 20, 0, 250, "'reader_writer_test_galera_hostgroup'"), std::make_tuple(2, 6, 10, 14, 1, 20, 0, 150, "'reader_writer_test_galera_hostgroup'"), - std::make_tuple(3, 7, 11, 15, 1, 20, 0, 350, "'reader_writer_test_galera_hostgroup'"), + std::make_tuple(1, 5, 9, 13, 1, 20, 0, 250, "'reader_writer_test_galera_hostgroup'"), }; std::vector insert_mysql_galera_hostgroup_queries {}; @@ -791,10 +791,10 @@ int main(int, char**) { "active, max_writers, writer_is_also_reader, max_transactions_behind) " "VALUES (%d, %d, %d, %d, %d, %d, %d, %d)"; std::vector> insert_group_replication_values { - std::make_tuple(0, 4, 8, 12, 1, 10, 0, 200), - std::make_tuple(1, 5, 9, 13, 1, 20, 0, 250), std::make_tuple(2, 6, 10, 14, 1, 20, 0, 150), + std::make_tuple(0, 4, 8, 12, 1, 10, 0, 200), std::make_tuple(3, 7, 11, 15, 1, 20, 0, 350), + std::make_tuple(1, 5, 9, 13, 1, 20, 0, 250), }; std::vector insert_mysql_group_replication_hostgroup_queries {}; @@ -908,10 +908,10 @@ int main(int, char**) { "active, max_writers, writer_is_also_reader, max_transactions_behind, comment) " "VALUES (%d, %d, %d, %d, %d, %d, %d, %d, %s)"; std::vector> insert_group_replication_values { - std::make_tuple(0, 4, 8, 12, 1, 10, 0, 200, "'reader_writer_test_group_replication_hostgroup'"), - std::make_tuple(1, 5, 9, 13, 1, 20, 0, 250, "'reader_writer_test_group_replication_hostgroup'"), std::make_tuple(2, 6, 10, 14, 1, 20, 0, 150, "'reader_writer_test_group_replication_hostgroup'"), std::make_tuple(3, 7, 11, 15, 1, 20, 0, 350, "'reader_writer_test_group_replication_hostgroup'"), + std::make_tuple(1, 5, 9, 13, 1, 20, 0, 250, "'reader_writer_test_group_replication_hostgroup'"), + std::make_tuple(0, 4, 8, 12, 1, 10, 0, 200, "'reader_writer_test_group_replication_hostgroup'") }; std::vector insert_mysql_group_replication_hostgroup_queries {}; @@ -1174,10 +1174,10 @@ int main(int, char**) { "check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks) " "VALUES (%d, %d, %d, %d, '%s', %d, %d, %d, %d, %d, %d, %d, %d)"; std::vector> insert_aws_aurora_values { - std::make_tuple(0, 4, 1, 3306, ".test_domain0", 10000, 2000, 2000, 0, 1, 50, 100, 1), - std::make_tuple(1, 5, 1, 3307, ".test_domain1", 10001, 2001, 2001, 0, 2, 50, 100, 1), std::make_tuple(2, 6, 1, 3308, ".test_domain2", 10002, 2002, 2002, 0, 3, 50, 100, 1), std::make_tuple(3, 7, 1, 3309, ".test_domain3", 10003, 2003, 2003, 0, 4, 50, 100, 1), + std::make_tuple(0, 4, 1, 3306, ".test_domain0", 10000, 2000, 2000, 0, 1, 50, 100, 1), + std::make_tuple(1, 5, 1, 3307, ".test_domain1", 10001, 2001, 2001, 0, 2, 50, 100, 1), }; std::vector insert_mysql_aws_aurora_hostgroup_queries {}; @@ -1301,10 +1301,10 @@ int main(int, char**) { "check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, comment) " "VALUES (%d, %d, %d, %d, '%s', %d, %d, %d, %d, %d, %d, %d, %d, '%s')"; std::vector> insert_aws_aurora_values { - std::make_tuple(0, 4, 1, 3306, ".test_domain0", 10000, 2000, 2000, 0, 1, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup"), + std::make_tuple(3, 7, 1, 3309, ".test_domain3", 10003, 2003, 2003, 0, 4, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup"), std::make_tuple(1, 5, 1, 3307, ".test_domain1", 10001, 2001, 2001, 0, 2, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup"), std::make_tuple(2, 6, 1, 3308, ".test_domain2", 10002, 2002, 2002, 0, 3, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup"), - std::make_tuple(3, 7, 1, 3309, ".test_domain3", 10003, 2003, 2003, 0, 4, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup"), + std::make_tuple(0, 4, 1, 3306, ".test_domain0", 10000, 2000, 2000, 0, 1, 50, 100, 1, "reader_writer_test_aws_aurora_hostgroup"), }; std::vector insert_mysql_aws_aurora_hostgroup_queries {}; From 2e594bd6b1b9b67519957e489d0b03b6db7ffd96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Sun, 31 Jul 2022 21:44:15 +0200 Subject: [PATCH 20/26] Replace space in ProxySQL version Enterprise identifier in favor of dash --- lib/ProxySQL_Admin.cpp | 8 ++++---- lib/ProxySQL_Cluster.cpp | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 6a10bac79e..f8eb2289ae 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -4624,13 +4624,13 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { if (GloMyLdapAuth == nullptr) { query_length=strlen(q)+20+strlen(PROXYSQL_VERSION); } else { - query_length=strlen(q)+20+strlen(PROXYSQL_VERSION)+strlen(" Enterprise"); + query_length=strlen(q)+20+strlen(PROXYSQL_VERSION)+strlen("-Enterprise"); } query=(char *)l_alloc(query_length); if (GloMyLdapAuth == nullptr) { sprintf(query, q, PROXYSQL_VERSION); } else { - sprintf(query, q, PROXYSQL_VERSION" Enterprise"); + sprintf(query, q, PROXYSQL_VERSION"-Enterprise"); } goto __run_query; } @@ -4641,13 +4641,13 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { if (GloMyLdapAuth == nullptr) { query_length=strlen(q)+20+strlen(PROXYSQL_VERSION); } else { - query_length=strlen(q)+20+strlen(PROXYSQL_VERSION)+strlen(" Enterprise"); + query_length=strlen(q)+20+strlen(PROXYSQL_VERSION)+strlen("-Enterprise"); } query=(char *)l_alloc(query_length); if (GloMyLdapAuth == nullptr) { sprintf(query, q, PROXYSQL_VERSION); } else { - sprintf(query, q, PROXYSQL_VERSION" Enterprise"); + sprintf(query, q, PROXYSQL_VERSION"-Enterprise"); } goto __run_query; } diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index 5e1d8bc524..886373c6d5 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -114,7 +114,7 @@ void * ProxySQL_Cluster_Monitor_thread(void *args) { bool same_version = false; while ((row = mysql_fetch_row(result))) { if (row[0]) { - const char* PROXYSQL_VERSION_ = GloMyLdapAuth == nullptr ? PROXYSQL_VERSION : PROXYSQL_VERSION" Enterprise"; + const char* PROXYSQL_VERSION_ = GloMyLdapAuth == nullptr ? PROXYSQL_VERSION : PROXYSQL_VERSION"-Enterprise"; if (strcmp(row[0], PROXYSQL_VERSION_)==0) { proxy_info("Cluster: clustering with peer %s:%d . Remote version: %s . Self version: %s\n", node->hostname, node->port, row[0], PROXYSQL_VERSION_); same_version = true; From b2cf36429a6901cf4a6cf090ff682aee83b1b7a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Sun, 31 Jul 2022 22:13:54 +0200 Subject: [PATCH 21/26] Add 'computed checksum' info messages before performing pulling actions in Cluster --- lib/ProxySQL_Cluster.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index 886373c6d5..1f3e298ae7 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -900,6 +900,7 @@ void ProxySQL_Cluster::pull_mysql_query_rules_from_peer(const string& expected_c const uint64_t query_rules_hash = SQLite3_query_rules_resultset->raw_checksum() + SQLite3_query_rules_fast_routing_resultset->raw_checksum(); const string computed_checksum { get_checksum_from_hash(query_rules_hash) }; + proxy_info("Cluster: Computed checksum for MySQL Query Rules from peer %s:%d : %s\n", hostname, port, computed_checksum.c_str()); if (expected_checksum == computed_checksum) { @@ -1186,6 +1187,7 @@ void ProxySQL_Cluster::pull_mysql_users_from_peer(const string& expected_checksu const uint64_t users_raw_checksum = get_mysql_users_checksum(mysql_users_result, ldap_mapping_result, mysql_users_resultset); const string computed_checksum { get_checksum_from_hash(users_raw_checksum) }; + proxy_info("Cluster: Computed checksum for MySQL Users from peer %s:%d : %s\n", hostname, port, computed_checksum.c_str()); if (expected_checksum == computed_checksum) { update_mysql_users(mysql_users_result); @@ -1542,6 +1544,7 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer(const std::string& checksum, if (fetching_error == false) { const uint64_t servers_hash = compute_servers_tables_raw_checksum(results); const string computed_checksum { get_checksum_from_hash(servers_hash) }; + proxy_info("Cluster: Computed checksum for MySQL Servers from peer %s:%d : %s\n", hostname, port, computed_checksum.c_str()); if (computed_checksum == peer_checksum) { // No need to perform the conversion if checksums don't match @@ -1846,6 +1849,7 @@ void ProxySQL_Cluster::pull_global_variables_from_peer(const string& var_type, c uint64_t glovars_hash = mysql_raw_checksum(result); string computed_checksum { get_checksum_from_hash(glovars_hash) }; + proxy_info("Cluster: Computed checksum for %s Variables from peer %s:%d : %s\n", vars_type_str, hostname, port, computed_checksum.c_str()); if (expected_checksum == computed_checksum) { From 7a0a83bb20eb644045e005b82fdafa12186e9a63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Sun, 31 Jul 2022 22:14:50 +0200 Subject: [PATCH 22/26] Fix compilation adding constructor for 'incoming_servers_t' A constructor is required to overcome C++11 limitation of type no longer being an aggregate due to default member initialization. --- include/proxysql_admin.h | 3 +++ lib/ProxySQL_Admin.cpp | 16 ++++++++++++++++ lib/ProxySQL_Cluster.cpp | 4 ++-- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index e0bdd1dcc3..56c4264cc0 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -122,6 +122,9 @@ struct incoming_servers_t { SQLite3_result* incoming_group_replication_hostgroups = NULL; SQLite3_result* incoming_galera_hostgroups = NULL; SQLite3_result* incoming_aurora_hostgroups = NULL; + + incoming_servers_t(); + incoming_servers_t(SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*); }; class ProxySQL_Admin { diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index f8eb2289ae..e3cc1b4b0e 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -978,6 +978,22 @@ bool is_admin_command_or_alias(const std::vector& cmds, char *query return false; } +incoming_servers_t::incoming_servers_t() {} + +incoming_servers_t::incoming_servers_t( + SQLite3_result* incoming_mysql_servers, + SQLite3_result* incoming_replication_hostgroups, + SQLite3_result* incoming_group_replication_hostgroups, + SQLite3_result* incoming_galera_hostgroups, + SQLite3_result* incoming_aurora_hostgroups +) : + incoming_mysql_servers(incoming_mysql_servers), + incoming_replication_hostgroups(incoming_replication_hostgroups), + incoming_group_replication_hostgroups(incoming_group_replication_hostgroups), + incoming_galera_hostgroups(incoming_galera_hostgroups), + incoming_aurora_hostgroups(incoming_aurora_hostgroups) +{} + int ProxySQL_Test___GetDigestTable(bool reset, bool use_swap) { int r = 0; if (!GloQPro) return 0; diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index 1f3e298ae7..ac7789a3f9 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -1424,9 +1424,9 @@ uint64_t compute_servers_tables_raw_checksum(const vector& results) incoming_servers_t convert_servers_resultsets(const std::vector& results) { if (results.size() != sizeof(incoming_servers_t) / sizeof(void*)) { - return {}; + return incoming_servers_t {}; } else { - return { + return incoming_servers_t { get_SQLite3_resulset(results[0]).release(), get_SQLite3_resulset(results[1]).release(), get_SQLite3_resulset(results[2]).release(), From d8429ca71cf0fd086e996df51675ee6cbd50fdbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Mon, 1 Aug 2022 14:15:02 +0200 Subject: [PATCH 23/26] Replace resultset for Cluster request fetching 'runtime_mysql_servers' Query 'CLUSTER_QUERY_MYSQL_SERVERS' is now responded by 'runtime_mysql_servers' as it previously was. Documentation reasoning about the change has been added to 'MySQL_HostGroups_Manager::runtime_mysql_servers'. --- include/MySQL_HostGroups_Manager.h | 51 ++++++++++++++++++++++-------- include/proxysql_admin.h | 2 +- lib/MySQL_HostGroups_Manager.cpp | 43 ++++++++++++++++--------- lib/ProxySQL_Admin.cpp | 37 ++++++---------------- 4 files changed, 76 insertions(+), 57 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 5e3e0ee03e..d03cfa967a 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -54,6 +54,7 @@ "lag_num_checks INT NOT NULL CHECK (lag_num_checks >= 1 AND lag_num_checks <= 16) DEFAULT 1 , comment VARCHAR ," \ "UNIQUE (reader_hostgroup))" +#define MYHGM_GEN_ADMIN_RUNTIME_SERVERS "SELECT hostgroup_id, hostname, port, gtid_port, CASE status WHEN 0 THEN \"ONLINE\" WHEN 1 THEN \"SHUNNED\" WHEN 2 THEN \"OFFLINE_SOFT\" WHEN 3 THEN \"OFFLINE_HARD\" WHEN 4 THEN \"SHUNNED\" END status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers ORDER BY hostgroup_id, hostname, port" typedef std::unordered_map umap_mysql_errors; @@ -376,19 +377,35 @@ class MySQL_HostGroups_Manager { Galera_Info *get_galera_node_info(int hostgroup); /** - * @brief These resultset holds the latest values for 'incoming_*' tables used promoted servers to runtime. + * @brief This resultset holds the current values for 'runtime_mysql_servers' computed by either latest + * 'commit' or fetched from another Cluster node. It's also used by ProxySQL_Admin to respond to the + * intercepted query 'CLUSTER_QUERY_MYSQL_SERVERS'. + * @details This resultset can't right now just contain the value for 'incoming_mysql_servers' as with the + * rest of the intercepted resultset. This is due to 'runtime_mysql_servers' reconfigurations that can be + * triggered by monitoring actions like 'Galera' currently performs. These actions not only trigger status + * changes in the servers, but also re-generate the servers table via 'commit', thus generating a new + * checksum in the process. Because of this potential mismatch, the fetching server wouldn't be able to + * compute the proper checksum for the fetched 'runtime_mysql_servers' config. + * + * As previously stated, these reconfigurations are monitoring actions, they can't be packed or performed + * in a single action, since monitoring data is required, which may not be already present. This makes + * this a convergent, but iterative process, that can't be compressed into a single action. Using other + * servers 'runtime_mysql_servers' while fetching represents a best effort for avoiding these + * reconfigurations in servers that already holds the same monitoring conditions. If monitoring + * conditions are not the same, circular fetching is still possible due to the previously described + * scenario. + */ + SQLite3_result* runtime_mysql_servers; + /** + * @brief These resultset holds the latest values for 'incoming_*' tables used to promoted servers to runtime. * @details All these resultsets are used by 'Cluster' to fetch and promote the same configuration used in the * node across the whole cluster. For these, the queries: - * - 'CLUSTER_QUERY_MYSQL_SERVERS' * - 'CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS' * - 'CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS' * - 'CLUSTER_QUERY_MYSQL_GALERA' * - 'CLUSTER_QUERY_MYSQL_AWS_AURORA' - * Issued by 'Cluster' are intercepted by 'ProxySQL_Admin' and return the content of these resultsets. This is - * possible because 'incoming_*' tables represent the actual applied configuration to the node servers before - * reconfiguration ('commit') is triggered, making this process convergent on the supplied config. + * Issued by 'Cluster' are intercepted by 'ProxySQL_Admin' and return the content of these resultsets. */ - SQLite3_result* incoming_mysql_servers; SQLite3_result *incoming_replication_hostgroups; void generate_mysql_group_replication_hostgroups_table(); @@ -552,18 +569,24 @@ class MySQL_HostGroups_Manager { void wrlock(); void wrunlock(); int servers_add(SQLite3_result *resultset); - bool commit(const std::string& checksum = "", const time_t epoch = 0); + bool commit(SQLite3_result* runtime_mysql_servers = nullptr, const std::string& checksum = "", const time_t epoch = 0); /** - * @brief These setters/getter functions store and retrieve the currently hold resultset for the 'incoming_*' - * table set that have been loaded to runtime. Whose hold the config to be propagated by Cluster. + * @brief Store the resultset for the 'runtime_mysql_servers' table set that have been loaded to runtime. + * The store configuration is later used by Cluster to propagate current config. + * @param The resulset to be stored replacing the current one. + */ + void save_runtime_mysql_servers(SQLite3_result *); + /** + * @brief These setters/getter functions store and retrieve the currently hold resultset for the + * 'incoming_*' table set that have been loaded to runtime. The store configuration is later used by + * Cluster to propagate current config. * @param The resulset to be stored replacing the current one. */ - void set_incoming_mysql_servers(SQLite3_result *); - void set_incoming_replication_hostgroups(SQLite3_result *); - void set_incoming_group_replication_hostgroups(SQLite3_result *); - void set_incoming_galera_hostgroups(SQLite3_result *); - void set_incoming_aws_aurora_hostgroups(SQLite3_result *); + void save_incoming_replication_hostgroups(SQLite3_result *); + void save_incoming_group_replication_hostgroups(SQLite3_result *); + void save_incoming_galera_hostgroups(SQLite3_result *); + void save_incoming_aws_aurora_hostgroups(SQLite3_result *); SQLite3_result* get_current_mysql_servers_inner(); SQLite3_result* get_current_mysql_replication_hostgroups_inner(); diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index 56c4264cc0..5ba16f335a 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -117,7 +117,7 @@ struct admin_metrics_map_idx { extern int admin__web_verbosity; struct incoming_servers_t { - SQLite3_result* incoming_mysql_servers = NULL; + SQLite3_result* runtime_mysql_servers = NULL; SQLite3_result* incoming_replication_hostgroups = NULL; SQLite3_result* incoming_group_replication_hostgroups = NULL; SQLite3_result* incoming_galera_hostgroups = NULL; diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 3e919bd9c1..c7a0b24d6d 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -1410,7 +1410,7 @@ MySQL_HostGroups_Manager::MySQL_HostGroups_Manager() { mydb->execute(MYHGM_MYSQL_AWS_AURORA_HOSTGROUPS); mydb->execute("CREATE INDEX IF NOT EXISTS idx_mysql_servers_hostname_port ON mysql_servers (hostname,port)"); MyHostGroups=new PtrArray(); - incoming_mysql_servers=NULL; + runtime_mysql_servers=NULL; incoming_replication_hostgroups=NULL; incoming_group_replication_hostgroups=NULL; incoming_galera_hostgroups=NULL; @@ -1645,7 +1645,9 @@ SQLite3_result * MySQL_HostGroups_Manager::execute_query(char *query, char **err return resultset; } -bool MySQL_HostGroups_Manager::commit(const std::string& checksum, const time_t epoch) { +bool MySQL_HostGroups_Manager::commit( + SQLite3_result* runtime_mysql_servers, const std::string& checksum, const time_t epoch +) { unsigned long long curtime1=monotonic_time(); wrlock(); @@ -1947,6 +1949,18 @@ bool MySQL_HostGroups_Manager::commit(const std::string& checksum, const time_t SQLite3_result *resultset=NULL; char *query=(char *)"SELECT hostgroup_id, hostname, port, gtid_port, CASE status WHEN 0 OR 1 OR 4 THEN 0 ELSE status END status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers WHERE status<>3 ORDER BY hostgroup_id, hostname, port"; mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); + if (runtime_mysql_servers == nullptr) { + char* error = NULL; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = NULL; + + mydb->execute_statement(MYHGM_GEN_ADMIN_RUNTIME_SERVERS, &error, &cols, &affected_rows, &resultset); + save_runtime_mysql_servers(resultset); + } else { + save_runtime_mysql_servers(runtime_mysql_servers); + } + if (resultset) { if (resultset->rows_count) { if (init == false) { @@ -2594,9 +2608,8 @@ SQLite3_result * MySQL_HostGroups_Manager::dump_table_mysql_servers() { int cols=0; int affected_rows=0; SQLite3_result *resultset=NULL; - char *query=(char *)"SELECT hostgroup_id, hostname, port, gtid_port, CASE status WHEN 0 THEN \"ONLINE\" WHEN 1 THEN \"SHUNNED\" WHEN 2 THEN \"OFFLINE_SOFT\" WHEN 3 THEN \"OFFLINE_HARD\" WHEN 4 THEN \"SHUNNED\" END status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers ORDER BY hostgroup_id, hostname, port"; - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); - mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", MYHGM_GEN_ADMIN_RUNTIME_SERVERS); + mydb->execute_statement(MYHGM_GEN_ADMIN_RUNTIME_SERVERS, &error , &cols , &affected_rows , &resultset); wrunlock(); return resultset; } @@ -3789,15 +3802,15 @@ int MySQL_HostGroups_Manager::get_multiple_idle_connections(int _hid, unsigned l return num_conn_current; } -void MySQL_HostGroups_Manager::set_incoming_mysql_servers(SQLite3_result *s) { - if (incoming_mysql_servers) { - delete incoming_mysql_servers; - incoming_mysql_servers = nullptr; +void MySQL_HostGroups_Manager::save_runtime_mysql_servers(SQLite3_result *s) { + if (runtime_mysql_servers) { + delete runtime_mysql_servers; + runtime_mysql_servers = nullptr; } - incoming_mysql_servers=s; + runtime_mysql_servers=s; } -void MySQL_HostGroups_Manager::set_incoming_replication_hostgroups(SQLite3_result *s) { +void MySQL_HostGroups_Manager::save_incoming_replication_hostgroups(SQLite3_result *s) { if (incoming_replication_hostgroups) { delete incoming_replication_hostgroups; incoming_replication_hostgroups = nullptr; @@ -3805,7 +3818,7 @@ void MySQL_HostGroups_Manager::set_incoming_replication_hostgroups(SQLite3_resul incoming_replication_hostgroups=s; } -void MySQL_HostGroups_Manager::set_incoming_group_replication_hostgroups(SQLite3_result *s) { +void MySQL_HostGroups_Manager::save_incoming_group_replication_hostgroups(SQLite3_result *s) { if (incoming_group_replication_hostgroups) { delete incoming_group_replication_hostgroups; incoming_group_replication_hostgroups = NULL; @@ -3813,7 +3826,7 @@ void MySQL_HostGroups_Manager::set_incoming_group_replication_hostgroups(SQLite3 incoming_group_replication_hostgroups=s; } -void MySQL_HostGroups_Manager::set_incoming_galera_hostgroups(SQLite3_result *s) { +void MySQL_HostGroups_Manager::save_incoming_galera_hostgroups(SQLite3_result *s) { if (incoming_galera_hostgroups) { delete incoming_galera_hostgroups; incoming_galera_hostgroups = NULL; @@ -3821,7 +3834,7 @@ void MySQL_HostGroups_Manager::set_incoming_galera_hostgroups(SQLite3_result *s) incoming_galera_hostgroups=s; } -void MySQL_HostGroups_Manager::set_incoming_aws_aurora_hostgroups(SQLite3_result *s) { +void MySQL_HostGroups_Manager::save_incoming_aws_aurora_hostgroups(SQLite3_result *s) { if (incoming_aws_aurora_hostgroups) { delete incoming_aws_aurora_hostgroups; incoming_aws_aurora_hostgroups = NULL; @@ -3830,7 +3843,7 @@ void MySQL_HostGroups_Manager::set_incoming_aws_aurora_hostgroups(SQLite3_result } SQLite3_result* MySQL_HostGroups_Manager::get_current_mysql_servers_inner() { - return this->incoming_mysql_servers; + return this->runtime_mysql_servers; } SQLite3_result* MySQL_HostGroups_Manager::get_current_mysql_replication_hostgroups_inner() { diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index e3cc1b4b0e..249aee73cd 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -981,13 +981,13 @@ bool is_admin_command_or_alias(const std::vector& cmds, char *query incoming_servers_t::incoming_servers_t() {} incoming_servers_t::incoming_servers_t( - SQLite3_result* incoming_mysql_servers, + SQLite3_result* runtime_mysql_servers, SQLite3_result* incoming_replication_hostgroups, SQLite3_result* incoming_group_replication_hostgroups, SQLite3_result* incoming_galera_hostgroups, SQLite3_result* incoming_aurora_hostgroups ) : - incoming_mysql_servers(incoming_mysql_servers), + runtime_mysql_servers(runtime_mysql_servers), incoming_replication_hostgroups(incoming_replication_hostgroups), incoming_group_replication_hostgroups(incoming_group_replication_hostgroups), incoming_galera_hostgroups(incoming_galera_hostgroups), @@ -11890,41 +11890,24 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime( SQLite3_result *resultset_galera=NULL; SQLite3_result *resultset_aws_aurora=NULL; - SQLite3_result* incoming_mysql_servers = incoming_servers.incoming_mysql_servers; + SQLite3_result* runtime_mysql_servers = incoming_servers.runtime_mysql_servers; SQLite3_result* incoming_replication_hostgroups = incoming_servers.incoming_replication_hostgroups; SQLite3_result* incoming_group_replication_hostgroups = incoming_servers.incoming_group_replication_hostgroups; SQLite3_result* incoming_galera_hostgroups = incoming_servers.incoming_galera_hostgroups; SQLite3_result* incoming_aurora_hostgroups = incoming_servers.incoming_aurora_hostgroups; - // TODO: Fix ordering here and place mixed ordering in TEST char *query=(char *)"SELECT hostgroup_id,hostname,port,gtid_port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms,comment FROM main.mysql_servers ORDER BY hostgroup_id, hostname, port"; proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); - if (incoming_mysql_servers == nullptr) { + if (runtime_mysql_servers == nullptr) { admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset_servers); } else { - resultset_servers = incoming_mysql_servers; + resultset_servers = runtime_mysql_servers; } //MyHGH->wrlock(); if (error) { proxy_error("Error on %s : %s\n", query, error); } else { MyHGM->servers_add(resultset_servers); - size_t init_row_count = resultset_servers->rows_count; - size_t rm_rows_count = 0; - const auto is_offline_server = [&rm_rows_count] (SQLite3_row* row) { - if (strcasecmp(row->fields[4], "OFFLINE_HARD") == 0) { - rm_rows_count += 1; - return true; - } else { - return false; - } - }; - resultset_servers->rows.erase( - std::remove_if(resultset_servers->rows.begin(), resultset_servers->rows.end(), is_offline_server), - resultset_servers->rows.end() - ); - resultset_servers->rows_count = init_row_count - rm_rows_count; - MyHGM->set_incoming_mysql_servers(resultset_servers); } resultset=NULL; @@ -11954,7 +11937,7 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime( proxy_error("Error on %s : %s\n", query, error); } else { // Pass the resultset to MyHGM - MyHGM->set_incoming_replication_hostgroups(resultset_replication); + MyHGM->save_incoming_replication_hostgroups(resultset_replication); } //if (resultset) delete resultset; //resultset=NULL; @@ -11988,7 +11971,7 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime( proxy_error("Error on %s : %s\n", query, error); } else { // Pass the resultset to MyHGM - MyHGM->set_incoming_group_replication_hostgroups(resultset_group_replication); + MyHGM->save_incoming_group_replication_hostgroups(resultset_group_replication); } // support for Galera, table mysql_galera_hostgroups @@ -12019,7 +12002,7 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime( proxy_error("Error on %s : %s\n", query, error); } else { // Pass the resultset to MyHGM - MyHGM->set_incoming_galera_hostgroups(resultset_galera); + MyHGM->save_incoming_galera_hostgroups(resultset_galera); } // support for AWS Aurora, table mysql_aws_aurora_hostgroups @@ -12054,10 +12037,10 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime( proxy_error("Error on %s : %s\n", query, error); } else { // Pass the resultset to MyHGM - MyHGM->set_incoming_aws_aurora_hostgroups(resultset_aws_aurora); + MyHGM->save_incoming_aws_aurora_hostgroups(resultset_aws_aurora); } // commit all the changes - MyHGM->commit(checksum, epoch); + MyHGM->commit(runtime_mysql_servers, checksum, epoch); GloAdmin->save_mysql_servers_runtime_to_database(true); // clean up From fbd63122395e7b2065da93de7a018c7387bb6eac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Mon, 1 Aug 2022 16:56:37 +0200 Subject: [PATCH 24/26] Filter 'OFFLINE_HARD' servers for stored 'runtime_mysql_servers' resultset --- lib/MySQL_HostGroups_Manager.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index c7a0b24d6d..9287e34b1e 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -1956,6 +1956,25 @@ bool MySQL_HostGroups_Manager::commit( SQLite3_result* resultset = NULL; mydb->execute_statement(MYHGM_GEN_ADMIN_RUNTIME_SERVERS, &error, &cols, &affected_rows, &resultset); + + // Remove 'OFFLINE_HARD' servers since they are not relevant to propagate to other Cluster + // nodes, or relevant for checksum computation. + const size_t init_row_count = resultset->rows_count; + size_t rm_rows_count = 0; + const auto is_offline_server = [&rm_rows_count] (SQLite3_row* row) { + if (strcasecmp(row->fields[4], "OFFLINE_HARD") == 0) { + rm_rows_count += 1; + return true; + } else { + return false; + } + }; + resultset->rows.erase( + std::remove_if(resultset->rows.begin(), resultset->rows.end(), is_offline_server), + resultset->rows.end() + ); + resultset->rows_count = init_row_count - rm_rows_count; + save_runtime_mysql_servers(resultset); } else { save_runtime_mysql_servers(runtime_mysql_servers); From 37efc833e39c918706047120692251f86814f6b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Mon, 1 Aug 2022 16:59:00 +0200 Subject: [PATCH 25/26] Change wording in doc for 'MySQL_HostGroups_Manager::runtime_mysql_servers' --- include/MySQL_HostGroups_Manager.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index d03cfa967a..9014767fd1 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -390,8 +390,8 @@ class MySQL_HostGroups_Manager { * As previously stated, these reconfigurations are monitoring actions, they can't be packed or performed * in a single action, since monitoring data is required, which may not be already present. This makes * this a convergent, but iterative process, that can't be compressed into a single action. Using other - * servers 'runtime_mysql_servers' while fetching represents a best effort for avoiding these - * reconfigurations in servers that already holds the same monitoring conditions. If monitoring + * nodes 'runtime_mysql_servers' while fetching represents a best effort for avoiding these + * reconfigurations in nodes that already holds the same monitoring conditions. If monitoring * conditions are not the same, circular fetching is still possible due to the previously described * scenario. */ From f880338477285c98393525ebf92a2f03f1ca929f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Fri, 12 Aug 2022 13:09:40 +0200 Subject: [PATCH 26/26] Fix typos in some cluster queries definitions doc --- include/ProxySQL_Cluster.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/ProxySQL_Cluster.hpp b/include/ProxySQL_Cluster.hpp index c0d3dc9603..fde16de626 100644 --- a/include/ProxySQL_Cluster.hpp +++ b/include/ProxySQL_Cluster.hpp @@ -46,10 +46,10 @@ /* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_users'. See top comment for details. */ #define CLUSTER_QUERY_MYSQL_USERS "PROXY_SELECT username, password, use_ssl, default_hostgroup, default_schema, schema_locked, transaction_persistent, fast_forward, backend, frontend, max_connections, attributes, comment FROM runtime_mysql_users" -/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_users'. See top comment for details. */ +/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_query_rules'. See top comment for details. */ #define CLUSTER_QUERY_MYSQL_QUERY_RULES "PROXY_SELECT rule_id, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, cache_timeout, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, ok_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, attributes, comment FROM runtime_mysql_query_rules ORDER BY rule_id" -/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_users'. See top comment for details. */ +/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_query_rules_fast_routing'. See top comment for details. */ #define CLUSTER_QUERY_MYSQL_QUERY_RULES_FAST_ROUTING "PROXY_SELECT username, schemaname, flagIN, destination_hostgroup, comment FROM runtime_mysql_query_rules_fast_routing ORDER BY username, schemaname, flagIN" class ProxySQL_Checksum_Value_2: public ProxySQL_Checksum_Value {