diff --git a/.gitignore b/.gitignore index 539f686cff..5d528bedda 100644 --- a/.gitignore +++ b/.gitignore @@ -195,6 +195,9 @@ test/tap/tests/generate_set_session_csv test/tap/tests/set_testing-240.csv local_testing_datadir/ +test/cluster/node??/* + + #files generated during CI run proxysql-save.cfg test/tap/tests/test_cluster_sync_config/cluster_sync_node_stderr.txt diff --git a/include/MySQL_Session.h b/include/MySQL_Session.h index ed9308cc3d..8c06a91790 100644 --- a/include/MySQL_Session.h +++ b/include/MySQL_Session.h @@ -286,6 +286,7 @@ class MySQL_Session Session_Regex **match_regexes; + ProxySQL_Node_Address * proxysql_node_address; // this is used ONLY for Admin, and only if the other party is another proxysql instance part of a cluster bool use_ldap_auth; // this variable is relevant only if status == SETTING_VARIABLE diff --git a/include/ProxySQL_Cluster.hpp b/include/ProxySQL_Cluster.hpp index fb08900f6e..e0df3d77fe 100644 --- a/include/ProxySQL_Cluster.hpp +++ b/include/ProxySQL_Cluster.hpp @@ -45,6 +45,28 @@ class ProxySQL_Node_Metrics { } }; +class ProxySQL_Node_Address { + public: + pthread_t thrid; + uint64_t hash; // unused for now + char *uuid; + char *hostname; + char *admin_mysql_ifaces; + uint16_t port; + ProxySQL_Node_Address(char *h, uint16_t p) { + hostname = strdup(h); + admin_mysql_ifaces = NULL; + port = p; + uuid = NULL; + hash = 0; + } + ~ProxySQL_Node_Address() { + if (hostname) free(hostname); + if (uuid) free(uuid); + if (admin_mysql_ifaces) free(admin_mysql_ifaces); + } +}; + class ProxySQL_Node_Entry { private: uint64_t hash; @@ -228,6 +250,9 @@ class ProxySQL_Cluster { pthread_mutex_t update_mysql_users_mutex; pthread_mutex_t update_mysql_variables_mutex; pthread_mutex_t update_proxysql_servers_mutex; + // this records the interface that Admin is listening to + pthread_mutex_t admin_mysql_ifaces_mutex; + char *admin_mysql_ifaces; int cluster_check_interval_ms; int cluster_check_status_frequency; int cluster_mysql_query_rules_diffs_before_sync; @@ -254,6 +279,7 @@ class ProxySQL_Cluster { void get_credentials(char **, char **); void set_username(char *); void set_password(char *); + void set_admin_mysql_ifaces(char *); bool Update_Node_Metrics(char * _h, uint16_t _p, MYSQL_RES *_r, unsigned long long _response_time) { return nodes.Update_Node_Metrics(_h, _p, _r, _response_time); } @@ -285,6 +311,6 @@ class ProxySQL_Cluster { * - 'admin'. */ void pull_global_variables_from_peer(const std::string& type); - void pull_proxysql_servers_from_peer(); + void pull_proxysql_servers_from_peer(const char *expected_checksum); }; #endif /* CLASS_PROXYSQL_CLUSTER_H */ diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index 39d677654d..dbb2794b90 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -129,6 +129,8 @@ class ProxySQL_Admin { prometheus::SerialExposer serial_exposer; + std::mutex proxysql_servers_mutex; + void wrlock(); void wrunlock(); @@ -336,6 +338,8 @@ class ProxySQL_Admin { void load_admin_variables_to_runtime() { flush_admin_variables___database_to_runtime(admindb, true); } void save_admin_variables_from_runtime() { flush_admin_variables___runtime_to_database(admindb, true, true, false); } + void load_or_update_global_settings(SQLite3DB *); + void load_mysql_variables_to_runtime() { flush_mysql_variables___database_to_runtime(admindb, true); } void save_mysql_variables_from_runtime() { flush_mysql_variables___runtime_to_database(admindb, true, true, false); } diff --git a/include/proxysql_glovars.hpp b/include/proxysql_glovars.hpp index 6fd490a113..ddd19eb2f2 100644 --- a/include/proxysql_glovars.hpp +++ b/include/proxysql_glovars.hpp @@ -1,6 +1,9 @@ #ifndef __CLASS_PROXYSQL_GLOVARS_H #define __CLASS_PROXYSQL_GLOVARS_H +#define CLUSTER_SYNC_INTERFACES_ADMIN "('admin-mysql_ifaces','admin-restapi_port','admin-telnet_admin_ifaces','admin-telnet_stats_ifaces','admin-web_port')" +#define CLUSTER_SYNC_INTERFACES_MYSQL "('mysql-interfaces')" + #include #include @@ -45,14 +48,17 @@ class ProxySQL_GlobalVariables { bool configfile_open; char *__cmd_proxysql_config_file; char *__cmd_proxysql_datadir; + char *__cmd_proxysql_uuid; int __cmd_proxysql_nostart; int __cmd_proxysql_foreground; int __cmd_proxysql_gdbg; bool __cmd_proxysql_initial; bool __cmd_proxysql_reload; + bool cluster_sync_interfaces; // If true, also mysql-interfaces and admin-mysql_ifaces are synced. false by default char *__cmd_proxysql_admin_socket; char *config_file; char *datadir; + char *uuid; char *admindb; char *statsdb_disk; char *sqlite3serverdb; diff --git a/include/proxysql_structs.h b/include/proxysql_structs.h index b4481ed751..7780ee96f8 100644 --- a/include/proxysql_structs.h +++ b/include/proxysql_structs.h @@ -506,6 +506,7 @@ class MySQL_ResultSet; class Query_Processor_Output; class MySrvC; class Web_Interface_plugin; +class ProxySQL_Node_Address; #endif /* PROXYSQL_CLASSES */ //#endif /* __cplusplus */ diff --git a/include/query_processor.h b/include/query_processor.h index 75c36b884c..720616e61b 100644 --- a/include/query_processor.h +++ b/include/query_processor.h @@ -332,6 +332,7 @@ class Query_Processor { SQLite3_result * fast_routing_resultset; void load_fast_routing(SQLite3_result *resultset); SQLite3_result * get_current_query_rules_fast_routing(); + int get_current_query_rules_fast_routing_count(); int testing___find_HG_in_mysql_query_rules_fast_routing(char *username, char *schemaname, int flagIN); int testing___find_HG_in_mysql_query_rules_fast_routing_dual(char *username, char *schemaname, int flagIN); diff --git a/lib/MySQL_Session.cpp b/lib/MySQL_Session.cpp index 782c53581f..5b0bd9b520 100644 --- a/lib/MySQL_Session.cpp +++ b/lib/MySQL_Session.cpp @@ -556,6 +556,7 @@ MySQL_Session::MySQL_Session() { last_insert_id=0; // #1093 last_HG_affected_rows = -1; // #1421 : advanced support for LAST_INSERT_ID() + proxysql_node_address = NULL; use_ldap_auth = false; } @@ -654,6 +655,10 @@ MySQL_Session::~MySQL_Session() { __sync_sub_and_fetch(&GloMTH->status_variables.mirror_sessions_current,1); GloMTH->status_variables.p_gauge_array[p_th_gauge::mirror_concurrency]->Decrement(); } + if (proxysql_node_address) { + delete proxysql_node_address; + proxysql_node_address = NULL; + } } diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index ec4ca6269b..e83215447e 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -49,11 +49,13 @@ #include "platform.h" #include "microhttpd.h" +#include #ifdef WITHGCOV extern "C" void __gcov_dump(); extern "C" void __gcov_reset(); #endif + //#define MYSQL_THREAD_IMPLEMENTATION #define SELECT_VERSION_COMMENT "select @@version_comment limit 1" @@ -305,7 +307,7 @@ extern int ProxySQL_create_or_load_TLS(bool bootstrap, std::string& msg); #define PANIC(msg) { perror(msg); exit(EXIT_FAILURE); } pthread_mutex_t sock_mutex = PTHREAD_MUTEX_INITIALIZER; -pthread_mutex_t admin_mutex = PTHREAD_MUTEX_INITIALIZER; +//pthread_mutex_t admin_mutex = PTHREAD_MUTEX_INITIALIZER; pthread_mutex_t users_mutex = PTHREAD_MUTEX_INITIALIZER; pthread_mutex_t test_mysql_firewall_whitelist_mutex = PTHREAD_MUTEX_INITIALIZER; @@ -396,6 +398,8 @@ static int http_handler(void *cls, struct MHD_Connection *connection, const char #define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_FAST_ROUTING "CREATE TABLE mysql_query_rules_fast_routing (username VARCHAR NOT NULL , schemaname VARCHAR NOT NULL , flagIN INT NOT NULL DEFAULT 0 , destination_hostgroup INT CHECK (destination_hostgroup >= 0) NOT NULL , comment VARCHAR NOT NULL , PRIMARY KEY (username, schemaname, flagIN) )" +#define ADMIN_SQLITE_TABLE_GLOBAL_SETTINGS "CREATE TABLE global_settings (variable_name VARCHAR NOT NULL PRIMARY KEY , variable_value VARCHAR NOT NULL)" + #define ADMIN_SQLITE_TABLE_GLOBAL_VARIABLES "CREATE TABLE global_variables (variable_name VARCHAR NOT NULL PRIMARY KEY , variable_value VARCHAR NOT NULL)" #define ADMIN_SQLITE_RUNTIME_GLOBAL_VARIABLES "CREATE TABLE runtime_global_variables (variable_name VARCHAR NOT NULL PRIMARY KEY , variable_value VARCHAR NOT NULL)" @@ -534,6 +538,8 @@ static int http_handler(void *cls, struct MHD_Connection *connection, const char #define ADMIN_SQLITE_TABLE_RUNTIME_PROXYSQL_SERVERS "CREATE TABLE runtime_proxysql_servers (hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 6032 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostname, port) )" +#define STATS_SQLITE_TABLE_PROXYSQL_SERVERS_CLIENTS_STATUS "CREATE TABLE stats_proxysql_servers_clients_status (uuid VARCHAR NOT NULL , hostname VARCHAR NOT NULL , port INT NOT NULL , admin_mysql_ifaces VARCHAR NOT NULL , last_seen_at INT NOT NULL , PRIMARY KEY (uuid, hostname, port) )" + #define STATS_SQLITE_TABLE_PROXYSQL_SERVERS_STATUS "CREATE TABLE stats_proxysql_servers_status (hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 6032 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 0 , master VARCHAR NOT NULL , global_version INT NOT NULL , check_age_us INT NOT NULL , ping_time_us INT NOT NULL, checks_OK INT NOT NULL , checks_ERR INT NOT NULL , PRIMARY KEY (hostname, port) )" #define STATS_SQLITE_TABLE_PROXYSQL_SERVERS_METRICS "CREATE TABLE stats_proxysql_servers_metrics (hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 6032 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , response_time_ms INT NOT NULL , Uptime_s INT NOT NULL , last_check_ms INT NOT NULL , Queries INT NOT NULL , Client_Connections_connected INT NOT NULL , Client_Connections_created INT NOT NULL , PRIMARY KEY (hostname, port) )" @@ -1364,6 +1370,65 @@ bool admin_handler_command_kill_connection(char *query_no_space, unsigned int qu * return true if the command is not a valid one and needs to be executed by SQLite (that will return an error) */ bool admin_handler_command_proxysql(char *query_no_space, unsigned int query_no_space_length, MySQL_Session *sess, ProxySQL_Admin *pa) { + if (!(strncasecmp("PROXYSQL CLUSTER_NODE_UUID ", query_no_space, strlen("PROXYSQL CLUSTER_NODE_UUID ")))) { + int l = strlen("PROXYSQL CLUSTER_NODE_UUID "); + if (sess->client_myds->addr.port == 0) { + proxy_warning("Received PROXYSQL CLUSTER_NODE_UUID not from TCP socket. Exiting client\n"); + SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"Received PROXYSQL CLUSTER_NODE_UUID not from TCP socket"); + sess->client_myds->shut_soft(); + return false; + } + if (query_no_space_length >= l+36+2) { + uuid_t uu; + char *A_uuid = NULL; + char *B_interface = NULL; + c_split_2(query_no_space+l, " ", &A_uuid, &B_interface); // we split the value + if (uuid_parse(A_uuid, uu)==0 && B_interface && strlen(B_interface)) { + proxy_info("Received PROXYSQL CLUSTER_NODE_UUID from %s:%d : %s\n", sess->client_myds->addr.addr, sess->client_myds->addr.port, query_no_space+l); + if (sess->proxysql_node_address==NULL) { + sess->proxysql_node_address = new ProxySQL_Node_Address(sess->client_myds->addr.addr, sess->client_myds->addr.port); + sess->proxysql_node_address->uuid = strdup(A_uuid); + if (sess->proxysql_node_address->admin_mysql_ifaces) { + free(sess->proxysql_node_address->admin_mysql_ifaces); + } + sess->proxysql_node_address->admin_mysql_ifaces = strdup(B_interface); + proxy_info("Created new link with Cluster node %s:%d : %s at interface %s\n", sess->client_myds->addr.addr, sess->client_myds->addr.port, A_uuid, B_interface); + SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); + free(A_uuid); + free(B_interface); + return false; + } else { + if (strcmp(A_uuid, sess->proxysql_node_address->uuid)) { + proxy_error("Cluster node %s:%d is sending a new UUID : %s . Former UUID : %s . Exiting client\n", sess->client_myds->addr.addr, sess->client_myds->addr.port, A_uuid, sess->proxysql_node_address->uuid); + SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"Received PROXYSQL CLUSTER_NODE_UUID with a new UUID not matching the previous one"); + sess->client_myds->shut_soft(); + free(A_uuid); + free(B_interface); + return false; + } else { + proxy_info("Cluster node %s:%d is sending again its UUID : %s\n", sess->client_myds->addr.addr, sess->client_myds->addr.port, A_uuid); + SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); + free(A_uuid); + free(B_interface); + return false; + } + } + free(A_uuid); + free(B_interface); + return false; + } else { + proxy_warning("Received PROXYSQL CLUSTER_NODE_UUID from %s:%d with invalid format: %s . Exiting client\n", sess->client_myds->addr.addr, sess->client_myds->addr.port, query_no_space+l); + SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"Received PROXYSQL CLUSTER_NODE_UUID with invalid format"); + sess->client_myds->shut_soft(); + return false; + } + } else { + proxy_warning("Received PROXYSQL CLUSTER_NODE_UUID from %s:%d with invalid format: %s . Exiting client\n", sess->client_myds->addr.addr, sess->client_myds->addr.port, query_no_space+l); + SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"Received PROXYSQL CLUSTER_NODE_UUID with invalid format"); + sess->client_myds->shut_soft(); + return false; + } + } if (query_no_space_length==strlen("PROXYSQL READONLY") && !strncasecmp("PROXYSQL READONLY",query_no_space, query_no_space_length)) { // this command enables admin_read_only , so the admin module is in read_only mode proxy_info("Received PROXYSQL READONLY command\n"); @@ -2542,9 +2607,17 @@ bool admin_handler_command_load_or_save(char *query_no_space, unsigned int query ) { proxy_info("Received %s command\n", query_no_space); ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->mysql_servers_wrlock(); - SPA->load_proxysql_servers_to_runtime(); - SPA->mysql_servers_wrunlock(); + //SPA->mysql_servers_wrlock(); + // before calling load_proxysql_servers_to_runtime() we release + // sql_query_global_mutex to prevent a possible deadlock due to + // a race condition + // load_proxysql_servers_to_runtime() calls ProxySQL_Cluster::load_servers_list() + // that then calls ProxySQL_Cluster_Nodes::load_servers_list(), holding a mutex + pthread_mutex_unlock(&SPA->sql_query_global_mutex); + SPA->load_proxysql_servers_to_runtime(true); + // we re-acquired the mutex because it will be released by the calling function + pthread_mutex_lock(&SPA->sql_query_global_mutex); + //SPA->mysql_servers_wrunlock(); proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded ProxySQL servers to RUNTIME\n"); SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); return false; @@ -2560,9 +2633,17 @@ bool admin_handler_command_load_or_save(char *query_no_space, unsigned int query ) { proxy_info("Received %s command\n", query_no_space); ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->mysql_servers_wrlock(); + //SPA->mysql_servers_wrlock(); + // before save_proxysql_servers_runtime_to_database() we release + // sql_query_global_mutex to prevent a possible deadlock due to + // a race condition + // save_proxysql_servers_runtime_to_database() calls ProxySQL_Cluster::dump_table_proxysql_servers() + // that then holds a mutex + pthread_mutex_unlock(&SPA->sql_query_global_mutex); SPA->save_proxysql_servers_runtime_to_database(false); - SPA->mysql_servers_wrunlock(); + // we re-acquired the mutex because it will be released by the calling function + pthread_mutex_lock(&SPA->sql_query_global_mutex); + //SPA->mysql_servers_wrunlock(); proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved ProxySQL servers from RUNTIME\n"); SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); return false; @@ -3126,7 +3207,7 @@ bool ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsign } // if (stats_mysql_processlist || stats_mysql_connection_pool || stats_mysql_query_digest || stats_mysql_query_digest_reset) { if (refresh==true) { - pthread_mutex_lock(&admin_mutex); + //pthread_mutex_lock(&admin_mutex); //ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; if (stats_mysql_processlist) stats___mysql_processlist(); @@ -3207,9 +3288,15 @@ bool ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsign mysql_thread___hostgroup_manager_verbose = old_hostgroup_manager_verbose; } if (runtime_proxysql_servers) { - mysql_servers_wrlock(); + //mysql_servers_wrlock(); + // before save_proxysql_servers_runtime_to_database() we release + // sql_query_global_mutex to prevent a possible deadlock due to + // a race condition + // save_proxysql_servers_runtime_to_database() calls ProxySQL_Cluster::dump_table_proxysql_servers() + pthread_mutex_unlock(&SPA->sql_query_global_mutex); save_proxysql_servers_runtime_to_database(true); - mysql_servers_wrunlock(); + pthread_mutex_lock(&SPA->sql_query_global_mutex); + //mysql_servers_wrunlock(); } if (runtime_mysql_users) { save_mysql_users_runtime_to_database(true); @@ -3262,7 +3349,7 @@ bool ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsign GloMyMon->populate_monitor_mysql_server_aws_aurora_check_status(); } } - pthread_mutex_unlock(&admin_mutex); + //pthread_mutex_unlock(&admin_mutex); } if ( stats_mysql_processlist || stats_mysql_connection_pool || stats_mysql_connection_pool_reset || @@ -3567,6 +3654,20 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { } } } + + // if the client simply executes: + // SELECT COUNT(*) FROM runtime_mysql_query_rules_fast_routing + // we just return the count + if (strcmp("SELECT COUNT(*) FROM runtime_mysql_query_rules_fast_routing", query_no_space)==0) { + int cnt = GloQPro->get_current_query_rules_fast_routing_count(); + l_free(query_length,query); + char buf[256]; + sprintf(buf,"SELECT %d AS 'COUNT(*)'", cnt); + query=l_strdup(buf); + query_length=strlen(query)+1; + goto __run_query; + } + if (!strncasecmp("TRUNCATE ", query_no_space, strlen("TRUNCATE "))) { if (sess->session_type == PROXYSQL_SESSION_ADMIN) { // no stats if (strstr(query_no_space,"stats_mysql_query_digest")) { @@ -4036,9 +4137,9 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { if ((query_no_space_length>8) && (!strncasecmp("PROXYSQL ", query_no_space, 8))) { proxy_debug(PROXY_DEBUG_ADMIN, 4, "Received PROXYSQL command\n"); - pthread_mutex_lock(&admin_mutex); + //pthread_mutex_lock(&admin_mutex); run_query=admin_handler_command_proxysql(query_no_space, query_no_space_length, sess, pa); - pthread_mutex_unlock(&admin_mutex); + //pthread_mutex_unlock(&admin_mutex); goto __run_query; } if ((query_no_space_length>5) && ( (!strncasecmp("SAVE ", query_no_space, 5)) || (!strncasecmp("LOAD ", query_no_space, 5))) ) { @@ -4894,6 +4995,22 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { } __run_query: + if (sess->proxysql_node_address) { + if (sess->client_myds->active) { + time_t now = time(NULL); + string q = "INSERT OR REPLACE INTO stats_proxysql_servers_clients_status (uuid, hostname, port, admin_mysql_ifaces, last_seen_at) VALUES (\""; + q += sess->proxysql_node_address->uuid; + q += "\",\""; + q += sess->proxysql_node_address->hostname; + q += "\","; + q += std::to_string(sess->proxysql_node_address->port); + q += ",\""; + q += sess->proxysql_node_address->admin_mysql_ifaces; + q += "\","; + q += std::to_string(now) + ")"; + SPA->statsdb->execute(q.c_str()); + } + } if (run_query) { ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; if (sess->session_type == PROXYSQL_SESSION_ADMIN) { // no stats @@ -5660,6 +5777,7 @@ bool ProxySQL_Admin::init() { insert_into_tables_defs(tables_defs_config,"mysql_query_rules", ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES); insert_into_tables_defs(tables_defs_config,"mysql_query_rules_fast_routing", ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_FAST_ROUTING); insert_into_tables_defs(tables_defs_config,"global_variables", ADMIN_SQLITE_TABLE_GLOBAL_VARIABLES); + insert_into_tables_defs(tables_defs_config,"global_settings", ADMIN_SQLITE_TABLE_GLOBAL_SETTINGS); // the table is not required to be present on disk. Removing it due to #1055 insert_into_tables_defs(tables_defs_config,"mysql_collations", ADMIN_SQLITE_TABLE_MYSQL_COLLATIONS); insert_into_tables_defs(tables_defs_config,"scheduler", ADMIN_SQLITE_TABLE_SCHEDULER); @@ -5704,6 +5822,7 @@ bool ProxySQL_Admin::init() { insert_into_tables_defs(tables_defs_stats,"stats_proxysql_servers_checksums", STATS_SQLITE_TABLE_PROXYSQL_SERVERS_CHECKSUMS); insert_into_tables_defs(tables_defs_stats,"stats_proxysql_servers_metrics", STATS_SQLITE_TABLE_PROXYSQL_SERVERS_METRICS); insert_into_tables_defs(tables_defs_stats,"stats_proxysql_servers_status", STATS_SQLITE_TABLE_PROXYSQL_SERVERS_STATUS); + insert_into_tables_defs(tables_defs_stats,"stats_proxysql_servers_clients_status", STATS_SQLITE_TABLE_PROXYSQL_SERVERS_CLIENTS_STATUS); // upgrade mysql_servers if needed (upgrade from previous version) disk_upgrade_mysql_servers(); @@ -5748,6 +5867,8 @@ bool ProxySQL_Admin::init() { flush_admin_variables___runtime_to_database(configdb, false, false, false); flush_admin_variables___runtime_to_database(admindb, false, true, false); + load_or_update_global_settings(configdb); + __insert_or_replace_maintable_select_disktable(); // removing this line of code. It seems redundant @@ -6044,6 +6165,79 @@ int check_port_availability(int port_num, bool* port_free) { return ecode; } +void ProxySQL_Admin::load_or_update_global_settings(SQLite3DB *db) { + char *error=NULL; + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + char *q=(char *)"SELECT variable_name, variable_value FROM global_settings ORDER BY variable_name"; + db->execute_statement(q, &error , &cols , &affected_rows , &resultset); + if (error) { + proxy_error("Error on %s : %s\n", q, error); + } else { + // note: we don't lock, this is done only during bootstrap + { + char *uuid = NULL; + bool write_uuid = true; + // search for uuid + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + if (strcasecmp(r->fields[0],"uuid")==0) { + uuid = strdup(r->fields[1]); + uuid_t uu; + if (uuid) { + if (uuid_parse(uuid,uu)==0) { + // we successful read an UUID + } else { + proxy_error("Ignoring invalid UUID format in global_settings: %s\n", uuid); + free(uuid); + uuid = NULL; + } + } + } + } + if (uuid) { // we found an UUID in the DB + if (GloVars.uuid) { // an UUID is already defined + if (strcmp(uuid, GloVars.uuid)==0) { // the match + proxy_info("Using UUID: %s\n", uuid); + write_uuid = false; + } else { + // they do not match. The one on DB will be replaced + proxy_info("Using UUID: %s . Replacing UUID from database: %s\n", GloVars.uuid, uuid); + } + } else { + // the UUID already defined, so the one in the DB will be used + proxy_info("Using UUID from database: %s\n", uuid); + GloVars.uuid=strdup(uuid); + } + } else { + if (GloVars.uuid) { + // we will write the UUID in the DB + proxy_info("Using UUID: %s . Writing it to database\n", GloVars.uuid); + } else { + // UUID not defined anywhere, we will create a new one + uuid_t uu; + uuid_generate(uu); + char buf[40]; + uuid_unparse(uu, buf); + GloVars.uuid=strdup(buf); + proxy_info("Using UUID: %s , randomly generated. Writing it to database\n", GloVars.uuid); + } + } + if (write_uuid) { + std::string s = "INSERT OR REPLACE INTO global_settings VALUES (\"uuid\", \""; + s += GloVars.uuid; + s += "\")"; + db->execute(s.c_str()); + } + if (uuid) { + free(uuid); + uuid=NULL; + } + } + } +} + void ProxySQL_Admin::flush_admin_variables___database_to_runtime(SQLite3DB *db, bool replace) { proxy_debug(PROXY_DEBUG_ADMIN, 4, "Flushing ADMIN variables. Replace:%d\n", replace); char *error=NULL; @@ -6096,8 +6290,13 @@ void ProxySQL_Admin::flush_admin_variables___database_to_runtime(SQLite3DB *db, int cols=0; int affected_rows=0; SQLite3_result *resultset=NULL; - char *q=(char *)"SELECT variable_name, variable_value FROM runtime_global_variables WHERE variable_name LIKE 'admin-\%' ORDER BY variable_name"; - admindb->execute_statement(q, &error , &cols , &affected_rows , &resultset); + std::string q; + if (GloVars.cluster_sync_interfaces) { + q="SELECT variable_name, variable_value FROM runtime_global_variables WHERE variable_name LIKE 'admin-\%' ORDER BY variable_name"; + } else { + q="SELECT variable_name, variable_value FROM runtime_global_variables WHERE variable_name LIKE 'admin-\%' AND variable_name NOT IN " + string(CLUSTER_SYNC_INTERFACES_ADMIN) + " ORDER BY variable_name"; + } + admindb->execute_statement(q.c_str(), &error , &cols , &affected_rows , &resultset); uint64_t hash1 = resultset->raw_checksum(); uint32_t d32[2]; char buf[20]; @@ -6429,8 +6628,13 @@ void ProxySQL_Admin::flush_mysql_variables___database_to_runtime(SQLite3DB *db, int cols=0; int affected_rows=0; SQLite3_result *resultset=NULL; - char *q=(char *)"SELECT variable_name, variable_value FROM runtime_global_variables WHERE variable_name LIKE 'mysql-%' ORDER BY variable_name"; - admindb->execute_statement(q, &error , &cols , &affected_rows , &resultset); + std::string q; + q = "SELECT variable_name, variable_value FROM runtime_global_variables WHERE variable_name LIKE 'mysql-\%' AND variable_name NOT IN ('mysql-threads')"; + if (GloVars.cluster_sync_interfaces == false) { + q += " AND variable_name NOT IN " + string(CLUSTER_SYNC_INTERFACES_MYSQL); + } + q += " ORDER BY variable_name"; + admindb->execute_statement(q.c_str(), &error , &cols , &affected_rows , &resultset); uint64_t hash1 = resultset->raw_checksum(); uint32_t d32[2]; char buf[20]; @@ -7527,6 +7731,7 @@ bool ProxySQL_Admin::set_variable(char *name, char *value) { // this is the pub if (update_creds && variables.mysql_ifaces) { S_amll.update_ifaces(variables.mysql_ifaces, &S_amll.ifaces_mysql); } + GloProxyCluster->set_admin_mysql_ifaces(value); return true; } else { return false; @@ -12290,6 +12495,7 @@ unsigned long long ProxySQL_External_Scheduler::run_once() { newargs[i]=sr->args[i-1]; } newargs[0]=sr->filename; + proxy_info("Scheduler starting id: %u , filename: %s\n", sr->id, sr->filename); pid_t cpid; cpid = fork(); if (cpid == -1) { @@ -12396,6 +12602,7 @@ void ProxySQL_Admin::flush_proxysql_servers__from_disk_to_memory() { } void ProxySQL_Admin::save_proxysql_servers_runtime_to_database(bool _runtime) { + std::lock_guard lock(proxysql_servers_mutex); // make sure that the caller has called mysql_servers_wrlock() char *query=NULL; SQLite3_result *resultset=NULL; diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index 48a8b5fbfd..70c116fbeb 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -41,14 +41,6 @@ extern ProxySQL_Cluster * GloProxyCluster; extern ProxySQL_Admin *GloAdmin; extern MySQL_LDAP_Authentication* GloMyLdapAuth; -typedef struct _proxy_node_address_t { - pthread_t thrid; - uint64_t hash; // unused for now - char *hostname; - uint16_t port; -} proxy_node_address_t; - - void * ProxySQL_Cluster_Monitor_thread(void *args) { pthread_attr_t thread_attr; size_t tmp_stack_size=0; @@ -58,7 +50,7 @@ void * ProxySQL_Cluster_Monitor_thread(void *args) { } } - proxy_node_address_t * node = (proxy_node_address_t *)args; + ProxySQL_Node_Address * node = (ProxySQL_Node_Address *)args; mysql_thread_init(); pthread_detach(pthread_self()); @@ -96,8 +88,8 @@ void * ProxySQL_Cluster_Monitor_thread(void *args) { } } mysql_options(conn, MYSQL_OPT_CONNECT_TIMEOUT, &timeout); - mysql_options(conn, MYSQL_OPT_READ_TIMEOUT, &timeout_long); - mysql_options(conn, MYSQL_OPT_WRITE_TIMEOUT, &timeout); + //mysql_options(conn, MYSQL_OPT_READ_TIMEOUT, &timeout_long); + //mysql_options(conn, MYSQL_OPT_WRITE_TIMEOUT, &timeout); { unsigned char val = 1; mysql_options(conn, MYSQL_OPT_SSL_ENFORCE, &val); } //rc_conn = mysql_real_connect(conn, node->hostname, username, password, NULL, node->port, NULL, CLIENT_COMPRESS); // FIXME: add optional support for compression rc_conn = mysql_real_connect(conn, node->hostname, username, password, NULL, node->port, NULL, 0); @@ -117,6 +109,14 @@ void * ProxySQL_Cluster_Monitor_thread(void *args) { if (strcmp(row[0], PROXYSQL_VERSION)==0) { proxy_info("Cluster: clustering with peer %s:%d . Remote version: %s . Self version: %s\n", node->hostname, node->port, row[0], PROXYSQL_VERSION); same_version = true; + std::string q = "PROXYSQL CLUSTER_NODE_UUID "; + q += GloVars.uuid; + q += " "; + pthread_mutex_lock(&GloProxyCluster->admin_mysql_ifaces_mutex); + q += GloProxyCluster->admin_mysql_ifaces; + pthread_mutex_unlock(&GloProxyCluster->admin_mysql_ifaces_mutex); + proxy_info("Cluster: sending CLUSTER_NODE_UUID %s to peer %s:%d\n", GloVars.uuid, node->hostname, node->port); + rc_query = mysql_query(conn, q.c_str()); } else { proxy_warning("Cluster: different ProxySQL version with peer %s:%d . Remote: %s . Self: %s\n", node->hostname, node->port, row[0], PROXYSQL_VERSION); } @@ -162,6 +162,7 @@ void * ProxySQL_Cluster_Monitor_thread(void *args) { //query = query3; //unsigned long long before_query_time2=monotonic_time(); if (update_checksum) { + unsigned long long before_query_time=monotonic_time(); rc_query = mysql_query(conn,query3); if ( rc_query == 0 ) { query_error = NULL; @@ -184,7 +185,9 @@ void * ProxySQL_Cluster_Monitor_thread(void *args) { } else { query_error = query3; if (query_error_counter == 0) { - proxy_error("Cluster: unable to run query on %s:%d using user %s : %s\n", node->hostname, node->port , username, query_error); + unsigned long long after_query_time=monotonic_time(); + unsigned long long elapsed_time_us = (after_query_time - before_query_time); + proxy_error("Cluster: unable to run query on %s:%d using user %s after %llums : %s . Error: %s\n", node->hostname, node->port , username, elapsed_time_us/1000 , query_error, mysql_error(conn)); } if (++query_error_counter == QUERY_ERROR_RATE) query_error_counter = 0; } @@ -215,7 +218,9 @@ void * ProxySQL_Cluster_Monitor_thread(void *args) { } else { query_error = query2; if (query_error_counter == 0) { - proxy_error("Cluster: unable to run query on %s:%d using user %s : %s\n", node->hostname, node->port , username, query_error); + unsigned long long after_query_time=monotonic_time(); + unsigned long long elapsed_time_us = (after_query_time - before_query_time); + proxy_error("Cluster: unable to run query on %s:%d using user %s after %llums : %s . Error: %s\n", node->hostname, node->port , username, elapsed_time_us/1000 , query_error, mysql_error(conn)); } if (++query_error_counter == QUERY_ERROR_RATE) query_error_counter = 0; } @@ -224,7 +229,9 @@ void * ProxySQL_Cluster_Monitor_thread(void *args) { } else { query_error = query1; if (query_error_counter == 0) { - proxy_error("Cluster: unable to run query on %s:%d using user %s : %s\n", node->hostname, node->port , username, query_error); + unsigned long long after_query_time=monotonic_time(); + unsigned long long elapsed_time_us = (after_query_time - start_time); + proxy_error("Cluster: unable to run query on %s:%d using user %s after %llums : %s . Error: %s\n", node->hostname, node->port , username, elapsed_time_us/1000, query_error, mysql_error(conn)); } if (++query_error_counter == QUERY_ERROR_RATE) query_error_counter = 0; } @@ -265,8 +272,7 @@ void * ProxySQL_Cluster_Monitor_thread(void *args) { mysql_close(conn); } proxy_info("Cluster: closing thread for peer %s:%d\n", node->hostname, node->port); - free(node->hostname); - free(node); + delete node; //pthread_exit(0); mysql_thread_end(); //GloProxyCluster->thread_ending(node->thrid); @@ -595,6 +601,10 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { if (v->diff_check >= diff_mqr) { proxy_info("Cluster: detected a peer %s:%d with mysql_query_rules version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); GloProxyCluster->pull_mysql_query_rules_from_peer(); + if (strncmp(v->checksum, GloVars.checksums_values.mysql_query_rules.checksum, 20)==0) { + // we copied from the remote server, let's also copy the same epoch + GloVars.checksums_values.mysql_query_rules.epoch = v->epoch; + } } } if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_mqr*10)) == 0)) { @@ -622,6 +632,10 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { if (v->diff_check >= diff_ms) { proxy_info("Cluster: detected a peer %s:%d with mysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); GloProxyCluster->pull_mysql_servers_from_peer(); + if (strncmp(v->checksum, GloVars.checksums_values.mysql_servers.checksum, 20)==0) { + // we copied from the remote server, let's also copy the same epoch + GloVars.checksums_values.mysql_servers.epoch = v->epoch; + } } } if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_ms*10)) == 0)) { @@ -649,6 +663,10 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { if (v->diff_check >= diff_mu) { proxy_info("Cluster: detected a peer %s:%d with mysql_users version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); GloProxyCluster->pull_mysql_users_from_peer(); + if (strncmp(v->checksum, GloVars.checksums_values.mysql_users.checksum, 20)==0) { + // we copied from the remote server, let's also copy the same epoch + GloVars.checksums_values.mysql_users.epoch = v->epoch; + } } } if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_mu*10)) == 0)) { @@ -675,7 +693,15 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { ) { if (v->diff_check >= diff_ps) { proxy_info("Cluster: detected a peer %s:%d with proxysql_servers version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); - GloProxyCluster->pull_proxysql_servers_from_peer(); + // v->checksum will be destroyed when calling pull_proxysql_servers_from_peer() + // thus we need to copy it now + char *expected_checksum = strdup(v->checksum); + GloProxyCluster->pull_proxysql_servers_from_peer((const char *)expected_checksum); + if (strncmp(expected_checksum, GloVars.checksums_values.proxysql_servers.checksum, 20)==0) { + // we copied from the remote server, let's also copy the same epoch + GloVars.checksums_values.proxysql_servers.epoch = v->epoch; + } + free(expected_checksum); } } if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_ps*10)) == 0)) { @@ -704,6 +730,10 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { if (v->diff_check >= diff_mv) { proxy_info("Cluster: detected a peer %s:%d with mysql_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); GloProxyCluster->pull_global_variables_from_peer("mysql"); + if (strncmp(v->checksum, GloVars.checksums_values.mysql_variables.checksum, 20)==0) { + // we copied from the remote server, let's also copy the same epoch + GloVars.checksums_values.mysql_variables.epoch = v->epoch; + } } } if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_mv*10)) == 0)) { @@ -732,6 +762,10 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { if (v->diff_check >= diff_av) { proxy_info("Cluster: detected a peer %s:%d with admin_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); GloProxyCluster->pull_global_variables_from_peer("admin"); + if (strncmp(v->checksum, GloVars.checksums_values.admin_variables.checksum, 20)==0) { + // we copied from the remote server, let's also copy the same epoch + GloVars.checksums_values.admin_variables.epoch = v->epoch; + } } } if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_av*10)) == 0)) { @@ -760,6 +794,10 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { if (v->diff_check >= diff_lv) { proxy_info("Cluster: detected a peer %s:%d with ldap_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); GloProxyCluster->pull_global_variables_from_peer("ldap"); + if (strncmp(v->checksum, GloVars.checksums_values.ldap_variables.checksum, 20)==0) { + // we copied from the remote server, let's also copy the same epoch + GloVars.checksums_values.ldap_variables.epoch = v->epoch; + } } } if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_lv*10)) == 0)) { @@ -797,8 +835,8 @@ void ProxySQL_Cluster::pull_mysql_query_rules_from_peer() { unsigned int timeout = 1; unsigned int timeout_long = 60; mysql_options(conn, MYSQL_OPT_CONNECT_TIMEOUT, &timeout); - mysql_options(conn, MYSQL_OPT_READ_TIMEOUT, &timeout_long); - mysql_options(conn, MYSQL_OPT_WRITE_TIMEOUT, &timeout); + //mysql_options(conn, MYSQL_OPT_READ_TIMEOUT, &timeout_long); + //mysql_options(conn, MYSQL_OPT_WRITE_TIMEOUT, &timeout); { unsigned char val = 1; mysql_options(conn, MYSQL_OPT_SSL_ENFORCE, &val); } proxy_info("Cluster: Fetching MySQL Query Rules from peer %s:%d started\n", hostname, port); rc_conn = mysql_real_connect(conn, hostname, username, password, NULL, port, NULL, 0); @@ -813,6 +851,7 @@ void ProxySQL_Cluster::pull_mysql_query_rules_from_peer() { result2 = mysql_store_result(conn); proxy_info("Cluster: Fetching MySQL Query Rules from peer %s:%d completed\n", hostname, port); proxy_info("Cluster: Loading to runtime MySQL Query Rules from peer %s:%d\n", hostname, port); + pthread_mutex_lock(&GloAdmin->sql_query_global_mutex); GloAdmin->admindb->execute("DELETE FROM mysql_query_rules"); GloAdmin->admindb->execute("DELETE FROM mysql_query_rules_fast_routing"); MYSQL_ROW row; @@ -907,6 +946,7 @@ void ProxySQL_Cluster::pull_mysql_query_rules_from_peer() { } else { proxy_info("Cluster: NOT saving to disk MySQL Query Rules from peer %s:%d\n", hostname, port); } + pthread_mutex_unlock(&GloAdmin->sql_query_global_mutex); metrics.p_counter_array[p_cluster_counter::pulled_mysql_query_rules_success]->Increment(); } else { proxy_info("Cluster: Fetching MySQL Query Rules from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); @@ -960,8 +1000,8 @@ void ProxySQL_Cluster::pull_mysql_users_from_peer() { unsigned int timeout = 1; unsigned int timeout_long = 60; mysql_options(conn, MYSQL_OPT_CONNECT_TIMEOUT, &timeout); - mysql_options(conn, MYSQL_OPT_READ_TIMEOUT, &timeout_long); - mysql_options(conn, MYSQL_OPT_WRITE_TIMEOUT, &timeout); + //mysql_options(conn, MYSQL_OPT_READ_TIMEOUT, &timeout_long); + //mysql_options(conn, MYSQL_OPT_WRITE_TIMEOUT, &timeout); { unsigned char val = 1; mysql_options(conn, MYSQL_OPT_SSL_ENFORCE, &val); } proxy_info("Cluster: Fetching MySQL Users from peer %s:%d started\n", hostname, port); rc_conn = mysql_real_connect(conn, hostname, username, password, NULL, port, NULL, 0); @@ -1148,13 +1188,12 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer() { unsigned int timeout = 1; unsigned int timeout_long = 60; mysql_options(conn, MYSQL_OPT_CONNECT_TIMEOUT, &timeout); - mysql_options(conn, MYSQL_OPT_READ_TIMEOUT, &timeout_long); - mysql_options(conn, MYSQL_OPT_WRITE_TIMEOUT, &timeout); + //mysql_options(conn, MYSQL_OPT_READ_TIMEOUT, &timeout_long); + //mysql_options(conn, MYSQL_OPT_WRITE_TIMEOUT, &timeout); { unsigned char val = 1; mysql_options(conn, MYSQL_OPT_SSL_ENFORCE, &val); } proxy_info("Cluster: Fetching MySQL Servers from peer %s:%d started. Expected checksum %s\n", hostname, port, peer_checksum); rc_conn = mysql_real_connect(conn, hostname, username, password, NULL, port, NULL, 0); if (rc_conn) { - GloAdmin->mysql_servers_wrlock(); std::vector results {}; // Server query messages @@ -1266,6 +1305,7 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer() { proxy_info("Cluster: Fetching checksum for MySQL Servers from peer %s:%d successful. Checksum: %s\n", hostname, port, checks); // sync mysql_servers proxy_info("Cluster: Writing mysql_servers table\n"); + GloAdmin->mysql_servers_wrlock(); GloAdmin->admindb->execute("DELETE FROM mysql_servers"); MYSQL_ROW row; char *q=(char *)"INSERT INTO mysql_servers (hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment) VALUES (%s, \"%s\", %s, %s, %s, \"%s\", %s, %s, %s, %s, %s, '%s')"; @@ -1441,6 +1481,7 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer() { } else { proxy_info("Cluster: Not saving to disk MySQL Servers from peer %s:%d failed.\n", hostname, port); } + GloAdmin->mysql_servers_wrunlock(); } // free results @@ -1450,7 +1491,6 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer() { metrics.p_counter_array[p_cluster_counter::pulled_mysql_servers_success]->Increment(); } - GloAdmin->mysql_servers_wrunlock(); } else { proxy_info("Cluster: Fetching MySQL Servers from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); metrics.p_counter_array[p_cluster_counter::pulled_mysql_servers_failure]->Increment(); @@ -1521,8 +1561,8 @@ void ProxySQL_Cluster::pull_global_variables_from_peer(const std::string& var_ty unsigned int timeout = 1; unsigned int timeout_long = 60; mysql_options(conn, MYSQL_OPT_CONNECT_TIMEOUT, &timeout); - mysql_options(conn, MYSQL_OPT_READ_TIMEOUT, &timeout_long); - mysql_options(conn, MYSQL_OPT_WRITE_TIMEOUT, &timeout); + //mysql_options(conn, MYSQL_OPT_READ_TIMEOUT, &timeout_long); + //mysql_options(conn, MYSQL_OPT_WRITE_TIMEOUT, &timeout); { unsigned char val = 1; mysql_options(conn, MYSQL_OPT_SSL_ENFORCE, &val); } proxy_info("Cluster: Fetching %s variables from peer %s:%d started\n", vars_type_str, hostname, port); rc_conn = mysql_real_connect(conn, hostname, username, password, NULL, port, NULL, 0); @@ -1530,12 +1570,33 @@ void ProxySQL_Cluster::pull_global_variables_from_peer(const std::string& var_ty if (rc_conn) { std::string s_query = ""; string_format("SELECT * FROM runtime_global_variables WHERE variable_name LIKE '%s-%%'", s_query, var_type.c_str()); + if (var_type == "mysql") { + s_query += " AND variable_name NOT IN ('mysql-threads')"; + } + if (GloVars.cluster_sync_interfaces == false) { + if (var_type == "admin") { + s_query += " AND variable_name NOT IN " + string(CLUSTER_SYNC_INTERFACES_ADMIN); + } else if (var_type == "mysql") { + s_query += " AND variable_name NOT IN " + string(CLUSTER_SYNC_INTERFACES_MYSQL); + } + } mysql_query(conn, s_query.c_str()); if (rc_query == 0) { MYSQL_RES *result = mysql_store_result(conn); std::string d_query = ""; - string_format("DELETE FROM runtime_global_variables WHERE variable_name LIKE '%s-%%'", d_query, var_type.c_str()); + // remember that we read from runtime_global_variables but write into global_variables + string_format("DELETE FROM global_variables WHERE variable_name LIKE '%s-%%'", d_query, var_type.c_str()); + if (var_type == "mysql") { + s_query += " AND variable_name NOT IN ('mysql-threads')"; + } + if (GloVars.cluster_sync_interfaces == false) { + if (var_type == "admin") { + d_query += " AND variable_name NOT IN " + string(CLUSTER_SYNC_INTERFACES_ADMIN); + } else if (var_type == "mysql") { + d_query += " AND variable_name NOT IN " + string(CLUSTER_SYNC_INTERFACES_MYSQL); + } + } GloAdmin->admindb->execute(d_query.c_str()); MYSQL_ROW row; @@ -1603,7 +1664,7 @@ void ProxySQL_Cluster::pull_global_variables_from_peer(const std::string& var_ty pthread_mutex_unlock(&GloProxyCluster->update_mysql_variables_mutex); } -void ProxySQL_Cluster::pull_proxysql_servers_from_peer() { +void ProxySQL_Cluster::pull_proxysql_servers_from_peer(const char *expected_checksum) { char * hostname = NULL; uint16_t port = 0; pthread_mutex_lock(&GloProxyCluster->update_proxysql_servers_mutex); @@ -1624,45 +1685,69 @@ void ProxySQL_Cluster::pull_proxysql_servers_from_peer() { unsigned int timeout = 1; unsigned int timeout_long = 60; mysql_options(conn, MYSQL_OPT_CONNECT_TIMEOUT, &timeout); - mysql_options(conn, MYSQL_OPT_READ_TIMEOUT, &timeout_long); - mysql_options(conn, MYSQL_OPT_WRITE_TIMEOUT, &timeout); + //mysql_options(conn, MYSQL_OPT_READ_TIMEOUT, &timeout_long); + //mysql_options(conn, MYSQL_OPT_WRITE_TIMEOUT, &timeout); { unsigned char val = 1; mysql_options(conn, MYSQL_OPT_SSL_ENFORCE, &val); } - proxy_info("Cluster: Fetching ProxySQL Servers from peer %s:%d started\n", hostname, port); + proxy_info("Cluster: Fetching ProxySQL Servers from peer %s:%d started. Expected checksum: %s\n", hostname, port, expected_checksum); rc_conn = mysql_real_connect(conn, hostname, username, password, NULL, port, NULL, 0); if (rc_conn) { - rc_query = mysql_query(conn,"SELECT hostname, port, weight, comment FROM runtime_proxysql_servers"); + rc_query = mysql_query(conn,"SELECT hostname, port, weight, comment FROM runtime_proxysql_servers ORDER BY hostname, port"); if ( rc_query == 0 ) { + char **pta=(char **)malloc(sizeof(char *)*4); + SQLite3_result *result3=new SQLite3_result(4); + result3->add_column_definition(SQLITE_TEXT,"hostname"); + result3->add_column_definition(SQLITE_TEXT,"port"); + result3->add_column_definition(SQLITE_TEXT,"weight"); + result3->add_column_definition(SQLITE_TEXT,"comment"); MYSQL_RES *result = mysql_store_result(conn); - GloAdmin->admindb->execute("DELETE FROM proxysql_servers"); MYSQL_ROW row; - char *q=(char *)"INSERT INTO proxysql_servers (hostname, port, weight, comment) VALUES (\"%s\", %s, %s, '%s')"; while ((row = mysql_fetch_row(result))) { - int i; - int l=0; - for (i=0; i<3; i++) { - l+=strlen(row[i]); + for (int i=0; i<4; i++) { + pta[i] = row[i]; } - char *o=escape_string_single_quotes(row[3],false); - char *query = (char *)malloc(strlen(q)+i+strlen(o)+64); - - sprintf(query,q,row[0],row[1],row[2],o); - if (o!=row[3]) { // there was a copy - free(o); - } - GloAdmin->admindb->execute(query); - free(query); + result3->add_row(pta); } - mysql_free_result(result); - proxy_info("Cluster: Fetching ProxySQL Servers from peer %s:%d completed\n", hostname, port); - proxy_info("Cluster: Loading to runtime ProxySQL Servers from peer %s:%d\n", hostname, port); - GloAdmin->load_proxysql_servers_to_runtime(false); - if (GloProxyCluster->cluster_proxysql_servers_save_to_disk == true) { - proxy_info("Cluster: Saving to disk ProxySQL Servers from peer %s:%d\n", hostname, port); - GloAdmin->flush_proxysql_servers__from_memory_to_disk(); + free(pta); + uint64_t hash1 = result3->raw_checksum(); + uint32_t d32[2]; + char buf[20]; + memcpy(&d32, &hash1, sizeof(hash1)); + sprintf(buf,"0x%0X%0X", d32[0], d32[1]); + delete result3; + proxy_info("Cluster: Fetching ProxySQL Servers from peer %s:%d completed. Computed checksum: %s\n", hostname, port, buf); + if (strcmp(buf, expected_checksum)==0) { + mysql_data_seek(result,0); + GloAdmin->admindb->execute("DELETE FROM proxysql_servers"); + char *q=(char *)"INSERT INTO proxysql_servers (hostname, port, weight, comment) VALUES (\"%s\", %s, %s, '%s')"; + while ((row = mysql_fetch_row(result))) { + int i; + int l=0; + for (i=0; i<3; i++) { + l+=strlen(row[i]); + } + char *o=escape_string_single_quotes(row[3],false); + char *query = (char *)malloc(strlen(q)+i+strlen(o)+64); + sprintf(query,q,row[0],row[1],row[2],o); + if (o!=row[3]) { // there was a copy + free(o); + } + GloAdmin->admindb->execute(query); + free(query); + } + proxy_info("Cluster: Loading to runtime ProxySQL Servers from peer %s:%d\n", hostname, port); + GloAdmin->load_proxysql_servers_to_runtime(false); + if (GloProxyCluster->cluster_proxysql_servers_save_to_disk == true) { + proxy_info("Cluster: Saving to disk ProxySQL Servers from peer %s:%d\n", hostname, port); + GloAdmin->flush_proxysql_servers__from_memory_to_disk(); + } else { + proxy_info("Cluster: NOT saving to disk ProxySQL Servers from peer %s:%d\n", hostname, port); + } + metrics.p_counter_array[p_cluster_counter::pulled_proxysql_servers_success]->Increment(); } else { - proxy_info("Cluster: NOT saving to disk ProxySQL Servers from peer %s:%d\n", hostname, port); + proxy_info("Cluster: Fetching ProxySQL Servers from peer %s:%d failed: Checksum changed from %s to %s\n", hostname, port, expected_checksum, buf); + metrics.p_counter_array[p_cluster_counter::pulled_proxysql_servers_failure]->Increment(); } - metrics.p_counter_array[p_cluster_counter::pulled_proxysql_servers_success]->Increment(); + mysql_free_result(result); } else { proxy_info("Cluster: Fetching ProxySQL Servers from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn)); metrics.p_counter_array[p_cluster_counter::pulled_proxysql_servers_failure]->Increment(); @@ -1782,10 +1867,7 @@ void ProxySQL_Cluster_Nodes::load_servers_list(SQLite3_result *resultset, bool _ node = new ProxySQL_Node_Entry(h_, p_, w_ , c_); node->set_active(true); umap_proxy_nodes.insert(std::make_pair(hash_, node)); - proxy_node_address_t * a = (proxy_node_address_t *)malloc(sizeof(proxy_node_address_t)); - a->hash = 0; // usused for now - a->hostname = strdup(h_); - a->port = p_; + ProxySQL_Node_Address * a = new ProxySQL_Node_Address(h_, p_); pthread_attr_t attr; pthread_attr_init(&attr); pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); @@ -2805,6 +2887,9 @@ ProxySQL_Cluster::ProxySQL_Cluster() { pthread_mutex_init(&update_mysql_servers_mutex,NULL); pthread_mutex_init(&update_mysql_users_mutex,NULL); pthread_mutex_init(&update_proxysql_servers_mutex,NULL); + pthread_mutex_init(&update_mysql_variables_mutex,NULL); + pthread_mutex_init(&admin_mysql_ifaces_mutex,NULL); + admin_mysql_ifaces = strdup((char *)""); // always initialized cluster_username = strdup((char *)""); cluster_password = strdup((char *)""); cluster_check_interval_ms = 1000; @@ -2830,6 +2915,10 @@ ProxySQL_Cluster::~ProxySQL_Cluster() { free(cluster_password); cluster_password = NULL; } + if (admin_mysql_ifaces) { + free(admin_mysql_ifaces); + admin_mysql_ifaces = NULL; + } } // this function returns credentials to the caller, used by monitoring threads @@ -2854,6 +2943,13 @@ void ProxySQL_Cluster::set_password(char *_password) { pthread_mutex_unlock(&mutex); } +void ProxySQL_Cluster::set_admin_mysql_ifaces(char *value) { + pthread_mutex_lock(&admin_mysql_ifaces_mutex); + free(admin_mysql_ifaces); + admin_mysql_ifaces=strdup(value); + pthread_mutex_unlock(&admin_mysql_ifaces_mutex); +} + void ProxySQL_Cluster::print_version() { fprintf(stderr,"Standard ProxySQL Cluster rev. %s -- %s -- %s\n", PROXYSQL_CLUSTER_VERSION, __FILE__, __TIMESTAMP__); }; diff --git a/lib/ProxySQL_GloVars.cpp b/lib/ProxySQL_GloVars.cpp index d4cdf76ed1..6b8afb605e 100644 --- a/lib/ProxySQL_GloVars.cpp +++ b/lib/ProxySQL_GloVars.cpp @@ -6,6 +6,7 @@ #include #include "SpookyV2.h" #include +#include #include "MySQL_LDAP_Authentication.hpp" @@ -80,13 +81,16 @@ ProxySQL_GlobalVariables::ProxySQL_GlobalVariables() : confFile=NULL; __cmd_proxysql_config_file=NULL; __cmd_proxysql_datadir=NULL; + __cmd_proxysql_uuid=NULL; config_file=NULL; datadir=NULL; + uuid=NULL; configfile_open=false; __cmd_proxysql_initial=false; __cmd_proxysql_reload=false; + cluster_sync_interfaces=false; statuses.stack_memory_mysql_threads = 0; statuses.stack_memory_admin_threads = 0; @@ -144,6 +148,7 @@ ProxySQL_GlobalVariables::ProxySQL_GlobalVariables() : opt->add((const char *)"",0,0,0,(const char *)"Do not restart ProxySQL if crashes",(const char *)"-e",(const char *)"--exit-on-error"); opt->add((const char *)"~/proxysql.cnf",0,1,0,(const char *)"Configuration file",(const char *)"-c",(const char *)"--config"); opt->add((const char *)"",0,1,0,(const char *)"Datadir",(const char *)"-D",(const char *)"--datadir"); + opt->add((const char *)"",0,1,0,(const char *)"UUID",(const char *)"-U",(const char *)"--uuid"); opt->add((const char *)"",0,0,0,(const char *)"Rename/empty database file",(const char *)"--initial"); opt->add((const char *)"",0,0,0,(const char *)"Merge config file into database file",(const char *)"--reload"); #ifdef IDLE_THREADS @@ -209,6 +214,19 @@ void ProxySQL_GlobalVariables::process_opts_pre() { GloVars.__cmd_proxysql_datadir=strdup(datadir.c_str()); } + if (opt->isSet("-U")) { + std::string uuid; + opt->get("-U")->getString(uuid); + uuid_t uu; + if (uuid_parse(uuid.c_str(), uu)==0) { + // we successfully parsed an UUID + if (GloVars.__cmd_proxysql_uuid) free(GloVars.__cmd_proxysql_uuid); + GloVars.__cmd_proxysql_uuid=strdup(uuid.c_str()); + } else { + fprintf(stderr,"The UUID specified in the command line is invalid, ignoring it: %s\n", uuid.c_str()); + } + } + if (opt->isSet("--initial")) { __cmd_proxysql_initial=true; } @@ -325,6 +343,10 @@ void ProxySQL_GlobalVariables::process_opts_post() { free(glovars.proxy_datadir); glovars.proxy_datadir=strdup(GloVars.__cmd_proxysql_datadir); } + if (GloVars.__cmd_proxysql_uuid) { + free(GloVars.uuid); + GloVars.uuid=strdup(GloVars.__cmd_proxysql_uuid); + } if (GloVars.__cmd_proxysql_admin_socket) { free(glovars.proxy_admin_socket); glovars.proxy_admin_socket=strdup(GloVars.__cmd_proxysql_admin_socket); diff --git a/lib/Query_Processor.cpp b/lib/Query_Processor.cpp index da9c3e65f8..bd0c2a9b3d 100644 --- a/lib/Query_Processor.cpp +++ b/lib/Query_Processor.cpp @@ -840,6 +840,14 @@ SQLite3_result * Query_Processor::get_current_query_rules() { return result; } +int Query_Processor::get_current_query_rules_fast_routing_count() { + int result = 0; + pthread_rwlock_rdlock(&rwlock); + result = fast_routing_resultset->rows_count; + pthread_rwlock_unlock(&rwlock); + return result; +} + SQLite3_result * Query_Processor::get_current_query_rules_fast_routing() { proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Dumping current query rules fast_routing, using Global version %d\n", version); SQLite3_result *result=new SQLite3_result(5); diff --git a/src/Makefile b/src/Makefile index e0939a10f4..e7200e1d3b 100644 --- a/src/Makefile +++ b/src/Makefile @@ -121,13 +121,13 @@ endif NOJEMALLOC := $(shell echo $(NOJEMALLOC)) ifeq ($(NOJEMALLOC),1) -MYLIBS=-Wl,--export-dynamic -Wl,-Bstatic -lconfig -lproxysql -ldaemon -lconfig++ -lre2 -lpcrecpp -lpcre -lmariadbclient -lhttpserver -lmicrohttpd -linjection -lcurl -lssl -lcrypto -lev -Wl,-Bdynamic -lgnutls -lpthread -lm -lz -lrt -lprometheus-cpp-pull -lprometheus-cpp-core $(EXTRALINK) +MYLIBS=-Wl,--export-dynamic -Wl,-Bstatic -lconfig -lproxysql -ldaemon -lconfig++ -lre2 -lpcrecpp -lpcre -lmariadbclient -lhttpserver -lmicrohttpd -linjection -lcurl -lssl -lcrypto -lev -luuid -Wl,-Bdynamic -lgnutls -lpthread -lm -lz -lrt -lprometheus-cpp-pull -lprometheus-cpp-core $(EXTRALINK) else -MYLIBS=-Wl,--export-dynamic -Wl,-Bstatic -lconfig -lproxysql -ldaemon -ljemalloc -lconfig++ -lre2 -lpcrecpp -lpcre -lmariadbclient -lhttpserver -lmicrohttpd -linjection -lcurl -lssl -lcrypto -lev -Wl,-Bdynamic -lgnutls -lpthread -lm -lz -lrt -lprometheus-cpp-pull -lprometheus-cpp-core $(EXTRALINK) +MYLIBS=-Wl,--export-dynamic -Wl,-Bstatic -lconfig -lproxysql -ldaemon -ljemalloc -lconfig++ -lre2 -lpcrecpp -lpcre -lmariadbclient -lhttpserver -lmicrohttpd -linjection -lcurl -lssl -lcrypto -lev -luuid -Wl,-Bdynamic -lgnutls -lpthread -lm -lz -lrt -lprometheus-cpp-pull -lprometheus-cpp-core $(EXTRALINK) endif ifeq ($(UNAME_S),Darwin) - MYLIBS=-lssl -lre2 -lmariadbclient -lpthread -lm -lz -liconv -lcrypto -lcurl -lgnutls + MYLIBS=-lssl -lre2 -lmariadbclient -lpthread -lm -lz -liconv -lcrypto -lcurl -lgnutls -luuid else CURL_DIR=$(DEPS_PATH)/curl/curl IDIRS+= -L$(CURL_DIR)/include diff --git a/src/main.cpp b/src/main.cpp index b6c7c7f969..a4247b25fa 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -40,6 +40,8 @@ #include +#include + /* extern "C" MySQL_LDAP_Authentication * create_MySQL_LDAP_Authentication_func() { return NULL; @@ -500,14 +502,41 @@ void ProxySQL_Main_process_global_variables(int argc, const char **argv) { GloVars.errorlog = strdup(errorlog_path.c_str()); } } + if (root.exists("uuid")==true) { + string uuid; + bool rc; + rc=root.lookupValue("uuid", uuid); + if (rc==true) { + uuid_t uu; + if (uuid_parse(uuid.c_str(), uu)==0) { + if (GloVars.uuid == NULL) { + // it is not set yet, that means it wasn't specified on the cmdline + GloVars.uuid = strdup(uuid.c_str()); + } + } else { + proxy_error("The config file is configured with an invalid UUID: %s\n", uuid.c_str()); + } + } + } + // if cluster_sync_interfaces is true, interfaces variables are synced too + if (root.exists("cluster_sync_interfaces")==true) { + bool value_bool; + bool rc; + rc=root.lookupValue("cluster_sync_interfaces", value_bool); + if (rc==true) { + GloVars.cluster_sync_interfaces=value_bool; + } else { + proxy_error("The config file is configured with an invalid cluster_sync_interfaces\n"); + } + } if (root.exists("pidfile")==true) { string pidfile_path; bool rc; rc=root.lookupValue("pidfile", pidfile_path); if (rc==true) { GloVars.pid = strdup(pidfile_path.c_str()); - } - } + } + } if (root.exists("sqlite3_plugin")==true) { string sqlite3_plugin; bool rc; diff --git a/src/proxysql.cfg b/src/proxysql.cfg index ffe2f7a34e..cde6a70f31 100644 --- a/src/proxysql.cfg +++ b/src/proxysql.cfg @@ -4,6 +4,8 @@ # http://www.hyperrealm.com/libconfig/libconfig_manual.html#Configuration-File-Grammar # Grammar is also copied at the end of this file +uuid="9588556d-fc2d-48ac-9c48-201abab06768" +cluster_sync_interfaces=false restart_on_missing_heartbeats=10 datadir="/var/lib/proxysql" //execute_on_exit_failure="/path/to/script" diff --git a/test/cluster/check_all_nodes.bash b/test/cluster/check_all_nodes.bash new file mode 100644 index 0000000000..afae05d1e4 --- /dev/null +++ b/test/cluster/check_all_nodes.bash @@ -0,0 +1,19 @@ +#!/bin/bash +TABLES=(mysql_servers mysql_users mysql_query_rules mysql_query_rules_fast_routing global_variables proxysql_servers mysql_galera_hostgroups mysql_group_replication_hostgroups mysql_replication_hostgroups) + +ALL_TABLES=() + +for i in ${!TABLES[@]} ; do + ALL_TABLES+=(${TABLES[$i]}) + ALL_TABLES+=("runtime_"${TABLES[$i]}) +done + +#for i in ${!ALL_TABLES[@]} ; do +# echo "SELECT * FROM ${ALL_TABLES[$i]};" +#done + +for p in 6032 `seq 26001 26009` ; do + for i in ${!ALL_TABLES[@]} ; do + echo "SELECT COUNT(*) FROM ${ALL_TABLES[$i]};" + done | mysql -u admin -padmin -h 127.0.0.1 -P$p > /dev/null 2> /dev/null & +done diff --git a/test/cluster/confs/proxysql01.cfg b/test/cluster/confs/proxysql01.cfg new file mode 100644 index 0000000000..aa3bc8c2b5 --- /dev/null +++ b/test/cluster/confs/proxysql01.cfg @@ -0,0 +1,36 @@ +cluster_sync_interfaces=false + +admin_variables= +{ + admin_credentials="admin:admin;cluster1:secret1pass" + mysql_ifaces="0.0.0.0:26001" + cluster_username="cluster1" + cluster_password="secret1pass" +} + +mysql_variables= +{ + interfaces="0.0.0.0:36001" +} + +proxysql_servers = +( + { + hostname="127.0.0.1" + port=26001 + weight=0 + comment="proxysql01" + }, + { + hostname="127.0.0.1" + port=26002 + weight=0 + comment="proxysql02" + }, + { + hostname="127.0.0.1" + port=26003 + weight=0 + comment="proxysql03" + } +) diff --git a/test/cluster/confs/proxysql02.cfg b/test/cluster/confs/proxysql02.cfg new file mode 100644 index 0000000000..85beb06bc9 --- /dev/null +++ b/test/cluster/confs/proxysql02.cfg @@ -0,0 +1,36 @@ +cluster_sync_interfaces=false + +admin_variables= +{ + admin_credentials="admin:admin;cluster1:secret1pass" + mysql_ifaces="0.0.0.0:26002" + cluster_username="cluster1" + cluster_password="secret1pass" +} + +mysql_variables= +{ + interfaces="0.0.0.0:36002" +} + +proxysql_servers = +( + { + hostname="127.0.0.1" + port=26001 + weight=0 + comment="proxysql01" + }, + { + hostname="127.0.0.1" + port=26002 + weight=0 + comment="proxysql02" + }, + { + hostname="127.0.0.1" + port=26003 + weight=0 + comment="proxysql03" + } +) diff --git a/test/cluster/confs/proxysql03.cfg b/test/cluster/confs/proxysql03.cfg new file mode 100644 index 0000000000..4bec8f9a4c --- /dev/null +++ b/test/cluster/confs/proxysql03.cfg @@ -0,0 +1,36 @@ +cluster_sync_interfaces=false + +admin_variables= +{ + admin_credentials="admin:admin;cluster1:secret1pass" + mysql_ifaces="0.0.0.0:26003" + cluster_username="cluster1" + cluster_password="secret1pass" +} + +mysql_variables= +{ + interfaces="0.0.0.0:36003" +} + +proxysql_servers = +( + { + hostname="127.0.0.1" + port=26001 + weight=0 + comment="proxysql01" + }, + { + hostname="127.0.0.1" + port=26002 + weight=0 + comment="proxysql02" + }, + { + hostname="127.0.0.1" + port=26003 + weight=0 + comment="proxysql03" + } +) diff --git a/test/cluster/confs/proxysql04.cfg b/test/cluster/confs/proxysql04.cfg new file mode 100644 index 0000000000..37eb66274c --- /dev/null +++ b/test/cluster/confs/proxysql04.cfg @@ -0,0 +1,36 @@ +cluster_sync_interfaces=false + +admin_variables= +{ + admin_credentials="admin:admin;cluster1:secret1pass" + mysql_ifaces="0.0.0.0:26004" + cluster_username="cluster1" + cluster_password="secret1pass" +} + +mysql_variables= +{ + interfaces="0.0.0.0:36004" +} + +proxysql_servers = +( + { + hostname="127.0.0.1" + port=26001 + weight=0 + comment="proxysql01" + }, + { + hostname="127.0.0.1" + port=26002 + weight=0 + comment="proxysql02" + }, + { + hostname="127.0.0.1" + port=26003 + weight=0 + comment="proxysql03" + } +) diff --git a/test/cluster/confs/proxysql05.cfg b/test/cluster/confs/proxysql05.cfg new file mode 100644 index 0000000000..6a48b085e0 --- /dev/null +++ b/test/cluster/confs/proxysql05.cfg @@ -0,0 +1,36 @@ +cluster_sync_interfaces=false + +admin_variables= +{ + admin_credentials="admin:admin;cluster1:secret1pass" + mysql_ifaces="0.0.0.0:26005" + cluster_username="cluster1" + cluster_password="secret1pass" +} + +mysql_variables= +{ + interfaces="0.0.0.0:36005" +} + +proxysql_servers = +( + { + hostname="127.0.0.1" + port=26001 + weight=0 + comment="proxysql01" + }, + { + hostname="127.0.0.1" + port=26002 + weight=0 + comment="proxysql02" + }, + { + hostname="127.0.0.1" + port=26003 + weight=0 + comment="proxysql03" + } +) diff --git a/test/cluster/confs/proxysql06.cfg b/test/cluster/confs/proxysql06.cfg new file mode 100644 index 0000000000..70728f0df5 --- /dev/null +++ b/test/cluster/confs/proxysql06.cfg @@ -0,0 +1,36 @@ +cluster_sync_interfaces=false + +admin_variables= +{ + admin_credentials="admin:admin;cluster1:secret1pass" + mysql_ifaces="0.0.0.0:26006" + cluster_username="cluster1" + cluster_password="secret1pass" +} + +mysql_variables= +{ + interfaces="0.0.0.0:36006" +} + +proxysql_servers = +( + { + hostname="127.0.0.1" + port=26001 + weight=0 + comment="proxysql01" + }, + { + hostname="127.0.0.1" + port=26002 + weight=0 + comment="proxysql02" + }, + { + hostname="127.0.0.1" + port=26003 + weight=0 + comment="proxysql03" + } +) diff --git a/test/cluster/confs/proxysql07.cfg b/test/cluster/confs/proxysql07.cfg new file mode 100644 index 0000000000..d6f16179a4 --- /dev/null +++ b/test/cluster/confs/proxysql07.cfg @@ -0,0 +1,36 @@ +cluster_sync_interfaces=false + +admin_variables= +{ + admin_credentials="admin:admin;cluster1:secret1pass" + mysql_ifaces="0.0.0.0:26007" + cluster_username="cluster1" + cluster_password="secret1pass" +} + +mysql_variables= +{ + interfaces="0.0.0.0:36007" +} + +proxysql_servers = +( + { + hostname="127.0.0.1" + port=26001 + weight=0 + comment="proxysql01" + }, + { + hostname="127.0.0.1" + port=26002 + weight=0 + comment="proxysql02" + }, + { + hostname="127.0.0.1" + port=26003 + weight=0 + comment="proxysql03" + } +) diff --git a/test/cluster/confs/proxysql08.cfg b/test/cluster/confs/proxysql08.cfg new file mode 100644 index 0000000000..17d0fecb40 --- /dev/null +++ b/test/cluster/confs/proxysql08.cfg @@ -0,0 +1,36 @@ +cluster_sync_interfaces=false + +admin_variables= +{ + admin_credentials="admin:admin;cluster1:secret1pass" + mysql_ifaces="0.0.0.0:26008" + cluster_username="cluster1" + cluster_password="secret1pass" +} + +mysql_variables= +{ + interfaces="0.0.0.0:36008" +} + +proxysql_servers = +( + { + hostname="127.0.0.1" + port=26001 + weight=0 + comment="proxysql01" + }, + { + hostname="127.0.0.1" + port=26002 + weight=0 + comment="proxysql02" + }, + { + hostname="127.0.0.1" + port=26003 + weight=0 + comment="proxysql03" + } +) diff --git a/test/cluster/confs/proxysql09.cfg b/test/cluster/confs/proxysql09.cfg new file mode 100644 index 0000000000..cca81da812 --- /dev/null +++ b/test/cluster/confs/proxysql09.cfg @@ -0,0 +1,36 @@ +cluster_sync_interfaces=false + +admin_variables= +{ + admin_credentials="admin:admin;cluster1:secret1pass" + mysql_ifaces="0.0.0.0:26009" + cluster_username="cluster1" + cluster_password="secret1pass" +} + +mysql_variables= +{ + interfaces="0.0.0.0:36009" +} + +proxysql_servers = +( + { + hostname="127.0.0.1" + port=26001 + weight=0 + comment="proxysql01" + }, + { + hostname="127.0.0.1" + port=26002 + weight=0 + comment="proxysql02" + }, + { + hostname="127.0.0.1" + port=26003 + weight=0 + comment="proxysql03" + } +) diff --git a/test/cluster/install_scheduler.bash b/test/cluster/install_scheduler.bash new file mode 100644 index 0000000000..01c6a1d2ab --- /dev/null +++ b/test/cluster/install_scheduler.bash @@ -0,0 +1,8 @@ +#/bin/bash +cp -f check_all_nodes.bash /tmp/check_all_nodes.bash +chmod +x /tmp/check_all_nodes.bash +mysql -u admin -padmin -h 127.0.0.1 -P6032 -e "INSERT INTO scheduler (interval_ms, filename) VALUES (12000, '/tmp/check_all_nodes.bash'); LOAD SCHEDULER TO RUNTIME; SAVE SCHEDULER TO DISK;" +for i in 1 2 3; do +sleep 3 +mysql -u admin -padmin -h 127.0.0.1 -P2600$i -e "INSERT INTO scheduler (interval_ms, filename) VALUES (12000, '/tmp/check_all_nodes.bash'); LOAD SCHEDULER TO RUNTIME; SAVE SCHEDULER TO DISK;" +done diff --git a/test/cluster/rolling_restart.sh b/test/cluster/rolling_restart.sh new file mode 100755 index 0000000000..26760991c4 --- /dev/null +++ b/test/cluster/rolling_restart.sh @@ -0,0 +1,21 @@ +#/bin/bash +echo "IGNORE errors like 'Lost connection to MySQL server during query'" +echo "Rolling restarting the core nodes" +for i in `seq 1 3` ; do + echo " restarting node $i ... " + mysql -u admin -padmin -h 127.0.0.1 -P2600$i -e "PROXYSQL SHUTDOWN SLOW" + sleep 1 + ../../src/proxysql -M -D $PWD/node0$i -c confs/proxysql0$i.cfg 2> /dev/null + echo "Done!" + echo -n "Pause" + for j in `seq 1 3` ; do echo -n "." ; sleep 1 ; done ; echo +done + +echo "Rolling restarting the satellite nodes" +for i in `seq 4 9` ; do + echo " restarting node $i ... " + mysql -u admin -padmin -h 127.0.0.1 -P2600$i -e "PROXYSQL SHUTDOWN SLOW" + sleep 1 + ../../src/proxysql -M -D $PWD/node0$i -c confs/proxysql0$i.cfg 2> /dev/null + echo "Done!" +done diff --git a/test/cluster/start.sh b/test/cluster/start.sh new file mode 100755 index 0000000000..b8d318b188 --- /dev/null +++ b/test/cluster/start.sh @@ -0,0 +1,10 @@ +#/bin/bash +echo "Starting the core nodes" +../../src/proxysql -D $PWD/node01 -c confs/proxysql01.cfg +../../src/proxysql -D $PWD/node02 -c confs/proxysql02.cfg +../../src/proxysql -D $PWD/node03 -c confs/proxysql03.cfg + +echo "Starting the satellite nodes" +for i in `seq 4 9` ; do + ../../src/proxysql -D $PWD/node0$i -c confs/proxysql0$i.cfg +done diff --git a/test/cluster/stop.sh b/test/cluster/stop.sh new file mode 100755 index 0000000000..a8f10fcce5 --- /dev/null +++ b/test/cluster/stop.sh @@ -0,0 +1,5 @@ +#/bin/bash +echo "Stopping all nodes" +for i in `seq 1 9` ; do + mysql -u admin -padmin -h 127.0.0.1 -P2600$i -e "PROXYSQL SHUTDOWN SLOW" +done diff --git a/test/tap/tests/Makefile b/test/tap/tests/Makefile index 1db7c1a937..0da22825f1 100644 --- a/test/tap/tests/Makefile +++ b/test/tap/tests/Makefile @@ -73,7 +73,7 @@ OBJ=../../../src/obj/proxysql_global.o ../../../src/obj/main.o ../../../src/obj/ INCLUDEDIRS=-I../tap -I$(RE2_PATH) -I$(IDIR) -I$(JEMALLOC_IDIR) -I$(SQLITE3_DIR) -I$(MICROHTTPD_IDIR) -I$(LIBHTTPSERVER_IDIR) -I$(CURL_IDIR) -I$(DAEMONPATH_IDIR) -I$(MARIADB_IDIR) -I$(SSL_IDIR) -I$(JSON_IDIR) -I$(LIBCONFIG_IDIR) -I$(PROMETHEUS_IDIR) -I$(EV_IDIR) LDIRS=-L$(TAP_LIBDIR) -L$(LDIR) -L$(JEMALLOC_LDIR) $(LIBCONFIG_LDIR) -L$(RE2_PATH)/obj -L$(MARIADB_LDIR) -L$(DAEMONPATH_LDIR) -L$(PCRE_LDIR) -L$(MICROHTTPD_LDIR) -L$(LIBHTTPSERVER_LDIR) -L$(LIBINJECTION_LDIR) -L$(CURL_LDIR) -L$(EV_LDIR) -L$(SSL_LDIR) -L$(PROMETHEUS_LDIR) -MYLIBS=-Wl,--export-dynamic -Wl,-Bstatic -lconfig -ldaemon -ljemalloc -lconfig++ -lre2 -lpcrecpp -lpcre -lmariadbclient -lhttpserver -lmicrohttpd -linjection -lcurl -lssl -lcrypto -lev -Wl,-Bdynamic -lgnutls -lpthread -lm -lz -lrt $(EXTRALINK) -lprometheus-cpp-pull -lprometheus-cpp-core +MYLIBS=-Wl,--export-dynamic -Wl,-Bstatic -lconfig -lproxysql -ldaemon -ljemalloc -lconfig++ -lre2 -lpcrecpp -lpcre -lmariadbclient -lhttpserver -lmicrohttpd -linjection -lcurl -lssl -lcrypto -lev -luuid -Wl,-Bdynamic -lgnutls -lpthread -lm -lz -lrt $(EXTRALINK) -lprometheus-cpp-pull -lprometheus-cpp-core STATIC_LIBS= $(SSL_LDIR)/libssl.a $(SSL_LDIR)/libcrypto.a .PHONY: all diff --git a/test/tap/tests/kill_connection-t.cpp b/test/tap/tests/kill_connection-t.cpp index 3efb8712b9..ade613df06 100644 --- a/test/tap/tests/kill_connection-t.cpp +++ b/test/tap/tests/kill_connection-t.cpp @@ -123,6 +123,7 @@ int main(int argc, char** argv) { diag("Running: %s", s.c_str()); MYSQL_QUERY(mysql, s.c_str()); } + sleep(1); } else { int rc = run_q(mysql, "DO 1"); ok(rc != 0, "Connection killed"); diff --git a/test/tap/tests/kill_connection3-t.cpp b/test/tap/tests/kill_connection3-t.cpp index 5f653d6389..e7fd6bc42e 100644 --- a/test/tap/tests/kill_connection3-t.cpp +++ b/test/tap/tests/kill_connection3-t.cpp @@ -150,7 +150,7 @@ int main(int argc, char** argv) { rc = run_q(proxysql_admin, s.c_str()); ok(rc == 0 , "%s" , s.c_str()); } - + sleep(1); for (int i = 0; i < NUM_CONNS ; i++) { MYSQL * mysql = conns[i]; int rc = run_q(mysql, "DO 1"); diff --git a/test/tap/tests/test_cluster1-t.cpp b/test/tap/tests/test_cluster1-t.cpp new file mode 100644 index 0000000000..ce017e985f --- /dev/null +++ b/test/tap/tests/test_cluster1-t.cpp @@ -0,0 +1,259 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "tap.h" +#include "command_line.h" +#include "utils.h" + + +/* + * this test assumes that this proxysql instance is part of a 10 nodes cluster + * there are 4 core nodes and 6 satellite nodes + * 127.0.0.1:6032 : this proxy (core node 0) + * 127.0.0.1:26001 : core node1 + * 127.0.0.1:26002 : core node2 + * 127.0.0.1:26003 : core node3 + * 127.0.0.1:26004 : satellite node1 + * 127.0.0.1:26005 : satellite node2 + * 127.0.0.1:26006 : satellite node3 + * 127.0.0.1:26007 : satellite node4 + * 127.0.0.1:26008 : satellite node5 + * 127.0.0.1:26009 : satellite node6 +*/ + +int run_q(MYSQL *mysql, const char *q) { + MYSQL_QUERY(mysql,q); + return 0; +} + +void get_time(std::string& s) { + time_t __timer; + char __buffer[30]; + struct tm __tm_info; + time(&__timer); + localtime_r(&__timer, &__tm_info); + strftime(__buffer, 25, "%Y-%m-%d %H:%M:%S", &__tm_info); + s = std::string(__buffer); +} + + + +// we make the very simple assumptions that all proxies are running +// on 127.0.0.1 , therefore we only specify the ports +const std::vector cluster_ports = { + 6032, + 26001, + 26002, + 26003, + 26004, + 26005, + 26006, + 26007, + 26008, + 26009 +}; + +// these simply queries update a variable to make sure that a resync is triggered +const char * update_admin_variables_1 = "UPDATE global_variables SET variable_value=variable_value+1 WHERE variable_name='admin-refresh_interval'"; +const char * update_mysql_variables_1 = "UPDATE global_variables SET variable_value=variable_value+1 WHERE variable_name='mysql-monitor_connect_interval'"; +const char * update_mysql_query_rules_1 = "UPDATE mysql_query_rules SET comment = IFNULL(comment,'') || 'a'"; +const char * update_mysql_users_1 = "UPDATE mysql_users SET max_connections = max_connections + 1"; +const char * update_mysql_servers_1 = "UPDATE mysql_servers SET max_connections = max_connections + 1"; +std::vector conns; + + +int dumping_checksums_return_uniq(MYSQL_RES *res, std::set& checksums) { + MYSQL_ROW row; + while ((row = mysql_fetch_row(res))) { + diag("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s", row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7], row[8]); + std::string chk = row[5]; + checksums.insert(chk); + } + return checksums.size(); +} + + +int get_checksum(MYSQL *mysql, const std::string& name, std::string& value) { + std::string query = "SELECT checksum FROM runtime_checksums_values WHERE name='" + name + "'"; + MYSQL_QUERY(mysql,query.c_str()); + MYSQL_RES * res = mysql_store_result(mysql); + int rr = mysql_num_rows(res); + MYSQL_ROW row; + while ((row = mysql_fetch_row(res))) { + value = std::string(row[0]); + } + ok(rr == 1 && value.length() > 0, "Checksum for %s = %s" , name.c_str(), value.c_str()); + mysql_free_result(res); + if (rr == 1 && value.length() > 0) return 0; + return 1; +} + +int module_in_sync(MYSQL *mysql, const std::string& name, const std::string& value, int num_retries, int& i) { + std::string query = "SELECT hostname, port, name, version, epoch, checksum, changed_at, updated_at, diff_check FROM stats_proxysql_servers_checksums WHERE name='" + name + "'"; + int rc = 0; + while (i< num_retries && rc != 1) { + std::set checksums; + MYSQL_QUERY(mysql,query.c_str()); + MYSQL_RES * res = mysql_store_result(mysql); + std::string s; + get_time(s); + diag("%s: Dumping %s", s.c_str(), query.c_str()); + int rc = dumping_checksums_return_uniq(res, checksums); + mysql_free_result(res); + if (rc == 1) { + std::set::iterator it = checksums.begin(); + if (*it == value) { + return 0; + } + } + sleep(1); + i++; + } + return 1; +} + +int create_connections(CommandLine& cl) { + for (int i = 0; i < cluster_ports.size() ; i++) { + MYSQL * mysql = mysql_init(NULL); + if (!mysql) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(mysql)); + return exit_status(); + } + + mysql_ssl_set(mysql, NULL, NULL, NULL, NULL, NULL); + if (!mysql_real_connect(mysql, cl.host, cl.admin_username, cl.admin_password, NULL, cluster_ports[i], NULL, CLIENT_SSL|CLIENT_COMPRESS)) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(mysql)); + return exit_status(); + } + conns.push_back(mysql); + } + return 0; +} + +int trigger_sync_and_check(MYSQL *mysql, std::string modname, const char *update_query, const char *load_query) { + int rc; + std::string chk1; + std::string chk2; + get_checksum(mysql, modname , chk1); + MYSQL_QUERY(mysql, update_query); + MYSQL_QUERY(mysql, load_query); + get_checksum(mysql, modname, chk2); + ok(chk1 != chk2 , "%s checksums. Before: %s , after: %s", modname.c_str(), chk1.c_str(), chk2.c_str()); + int retries = 0; + rc = module_in_sync(mysql, modname, chk2, 30, retries); + ok (rc == 0, "Module %s %sin sync after %d seconds" , modname.c_str() , rc == 0 ? "" : "NOT " , retries); + for (int i = 4 ; i modules = { + "admin_variables", + "ldap_variables", + "mysql_variables", + "mysql_query_rules", + "mysql_servers", + "mysql_users", + "proxysql_servers", + }; + + std::string chk1; + std::string chk2; + get_checksum(proxysql_admin, "admin_variables", chk1); + // we set all the admin-cluster_xxxx_save_to_disk to false to prevent that resync affects our testing proxy + for (std::vector::iterator it = modules.begin() ; it != modules.end() ; it++) { + std::string q = "SET admin-cluster_" + *it + "_save_to_disk='false'"; + MYSQL_QUERY(proxysql_admin, q.c_str()); + } + MYSQL_QUERY(proxysql_admin, update_admin_variables_1); + MYSQL_QUERY(proxysql_admin, "LOAD ADMIN VARIABLES TO RUNTIME"); + get_checksum(proxysql_admin, "admin_variables", chk2); + ok(chk1 != chk2 , "admin_variables checksums. Before: %s , after: %s", chk1.c_str(), chk2.c_str()); + + get_checksum(proxysql_admin, "mysql_variables", chk1); + MYSQL_QUERY(proxysql_admin, "SET mysql-have_ssl='true'"); + MYSQL_QUERY(proxysql_admin, "SET mysql-have_compress='true'"); + MYSQL_QUERY(proxysql_admin, "SET mysql-auditlog_filename=\"proxy-audit\""); + MYSQL_QUERY(proxysql_admin, update_mysql_variables_1); + MYSQL_QUERY(proxysql_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); + get_checksum(proxysql_admin, "mysql_variables", chk2); + ok(chk1 != chk2 , "mysql_variables checksums. Before: %s , after: %s", chk1.c_str(), chk2.c_str()); + + + + MYSQL_RES* proxy_res; + int rc = 0; + rc = create_connections(cl); + if (rc != 0) { + return exit_status(); + } + ok(conns.size() == cluster_ports.size() , "Known nodes: %lu . Connected nodes: %lu", cluster_ports.size(), conns.size()); + + int retries = 0; + rc = module_in_sync(proxysql_admin, "mysql_variables", chk2, 30, retries); + ok (rc == 0, "Module mysql_variables %sin sync after %d seconds" , rc == 0 ? "" : "NOT " , retries); + + + // the workflow here is simple: + // for each proxy: + // get the checksum of the module + // update the module + // get the new checksum + // wait for all the other core nodes to sync + for (int i = 0; i < 4; i++) { + diag("Running changes on server node %d", i); + trigger_sync_and_check(conns[i], "admin_variables", update_admin_variables_1, "LOAD ADMIN VARIABLES TO RUNTIME"); + } + for (int i = 0; i < 4; i++) { + diag("Running changes on server node %d", i); + trigger_sync_and_check(conns[i], "mysql_variables", update_mysql_variables_1, "LOAD MYSQL VARIABLES TO RUNTIME"); + } + for (int i = 0; i < 4; i++) { + diag("Running changes on server node %d", i); + trigger_sync_and_check(conns[i], "mysql_query_rules", update_mysql_query_rules_1, "LOAD MYSQL QUERY RULES TO RUNTIME"); + } + for (int i = 0; i < 4; i++) { + diag("Running changes on server node %d", i); + trigger_sync_and_check(conns[i], "mysql_users", update_mysql_users_1, "LOAD MYSQL USERS TO RUNTIME"); + } + for (int i = 0; i < 4; i++) { + diag("Running changes on server node %d", i); + trigger_sync_and_check(conns[i], "mysql_servers", update_mysql_servers_1, "LOAD MYSQL SERVERS TO RUNTIME"); + } + + return exit_status(); +} diff --git a/test/tap/tests/test_cluster_sync-t.cpp b/test/tap/tests/test_cluster_sync-t.cpp index f121500820..feb097a516 100644 --- a/test/tap/tests/test_cluster_sync-t.cpp +++ b/test/tap/tests/test_cluster_sync-t.cpp @@ -1138,7 +1138,7 @@ int main(int, char**) { const char* t_update_admin_variables = "UPDATE global_variables SET variable_value='%s' WHERE variable_name='%s'"; std::vector> update_admin_variables_values { - std::make_tuple("admin-admin_credentials" , "admin:admin;radmin:radmin" ), + std::make_tuple("admin-admin_credentials" , "admin:admin;radmin:radmin;cluster1:secret1pass" ), std::make_tuple("admin-checksum_admin_variables" , "true" ), std::make_tuple("admin-checksum_mysql_query_rules" , "true" ), std::make_tuple("admin-checksum_mysql_servers" , "true" ), @@ -1162,17 +1162,17 @@ int main(int, char**) { // std::make_tuple("admin-cluster_password" , "" ), Known issue, can't clear // std::make_tuple("admin-debug" , "false" ), Should not be synced std::make_tuple("admin-hash_passwords" , "true" ), - std::make_tuple("admin-mysql_ifaces" , "0.0.0.0:6032" ), + // std::make_tuple("admin-mysql_ifaces" , "0.0.0.0:6032" ), // disabled because of cluster_sync_interfaces=false std::make_tuple("admin-prometheus_memory_metrics_interval" , "61" ), std::make_tuple("admin-read_only" , "false" ), std::make_tuple("admin-refresh_interval" , "2001" ), std::make_tuple("admin-restapi_enabled" , "false" ), - std::make_tuple("admin-restapi_port" , "6071" ), + // std::make_tuple("admin-restapi_port" , "6071" ), std::make_tuple("admin-stats_credentials" , "stats:stats" ), std::make_tuple("admin-vacuum_stats" , "true" ), // std::make_tuple("admin-version" , "2.1.0-231-gbc0963e3_DEBUG" ), This changes at runtime, but it's not stored - std::make_tuple("admin-web_enabled" , "false" ), - std::make_tuple("admin-web_port" , "6080" ) + std::make_tuple("admin-web_enabled" , "false" ) + // std::make_tuple("admin-web_port" , "6080" ) // disabled because of cluster_sync_interfaces=false }; std::vector update_admin_variables_queries {};