[plugin/group_replication] PS-8592: XCom connection stalled forever i…

…n read() syscall over network

https://jira.percona.com/browse/PS-8592

Description
-----------
GR suffered from problems caused by the security probes and network scanner
processes connecting to the group replication communication port. This usually
is not a problem, but poses a serious threat when another member tries to join
the cluster by initialting a connection to the member which is affected by
external processes using the port dedicated for group communication for longer
durations.

On such activites by external processes, the SSL enabled server stalled forever
on the SSL_accept() call waiting for handshake data. Below is the stacktrace:

    Thread 55 (Thread 0x7f7bb77ff700 (LWP 2198598)):
    #0 in read ()
    #1 in sock_read ()
    percona#2 in BIO_read ()
    percona#3 in ssl23_read_bytes ()
    percona#4 in ssl23_get_client_hello ()
    percona#5 in ssl23_accept ()
    percona#6 in xcom_tcp_server_startup(Xcom_network_provider*) ()

When the server stalled in the above path forever, it prohibited other members
to join the cluster resulting in the following messages on the joiner server's
logs.

    [ERROR] [MY-011640] [Repl] Plugin group_replication reported: 'Timeout on wait for view after joining group'
    [ERROR] [MY-011735] [Repl] Plugin group_replication reported: '[GCS] The member is already leaving or joining a group.'

Solution
--------
This patch adds two new variables

1. group_replication_xcom_ssl_socket_timeout

   It is a file-descriptor level timeout in seconds for both accept() and
   SSL_accept() calls when group replication is listening on the xcom port.
   When set to a valid value, say for example 5 seconds, both accept() and
   SSL_accept() return after 5 seconds. The default value has been set to 0
   (waits infinitely) for backward compatibility. This variable is effective
   only when GR is configred with SSL.

2. group_replication_xcom_ssl_accept_retries

   It defines the number of retries to be performed before closing the socket.
   For each retry the server thread calls SSL_accept()  with timeout defined by
   the group_replication_xcom_ssl_socket_timeout for the SSL handshake process
   once the connection has been accepted by the first accept() call. The
   default value has been set to 10. This variable is effective only when GR is
   configred with SSL.

Note:
- Both of the above variables are dynamically configurable, but will become
  effective only on START GROUP_REPLICATION.

-------------------------------------------------------------------------

PS-8844: Fix the failing main.mysqldump_gtid_purged

https://jira.percona.com/browse/PS-8844

This patch fixes the test failure of main.mysqldump_gtid_purged that
failed due to the uninitialized variable $redirect_stderr in the
start_proc_in_background.inc.

----------------------------------------------------------------------

PS-9218: Merge MySQL 8.4.0 (fix terminology in replication tests)

https://perconadev.atlassian.net/browse/PS-9218

mysql/mysql-server@44a77b5

mysql-test/include/start_proc_in_background.inc

@@ -0,0 +1,71 @@
+# ==== Purpose ====
+#
+# Start process in background. Optionally store its PID in the file
+#
+# This functionality cannot be achieved by mysqltest built-in --exec_in_background
+# because we need to start it like:
+# cmd param1 param2 > output.log & echo $! > cmd.pid
+#
+# The way to pass $! through Perl to mysqltest binary is to escape $! like \$!
+# This will prevent attemption to resolve $! variable on Perl layer, 
+# but will pass down \$! to mysqltest. 
+# mysqltest handles \$! literally (recognizes that $ has been escaped
+# and adds escape character to the constructed command, so we end up with:
+# cmd param1 param2 > output.log & echo \$! > cmd.pid
+#
+# ==== Usage ====
+#
+# --let $command = process_to_be_executed
+# [--let $command_opt = opt1 opt2 ...]
+# [--let $output_file = output_file]
+# [--let $pid_file = pid_file]
+# [--let $redirect_stderr = 0 | 1 ]
+# --source include/start_proc_in_backcground.inc
+#
+# Parameters:
+#   $command
+#     Process to be executed in background.
+#
+#   $command_opt
+#     Options to be passed to the process via command line at its startup.
+#
+#   $output_file
+#     Redirect process output to this file.
+#
+#   $pid_file
+#     Store started process PID in this file.
+#
+
+
+if (!$command)
+{
+  --die ERROR IN TEST: 'command' parameter not specified
+}
+
+--let $line = $command $command_opt
+
+if ($output_file)
+{
+  if ($redirect_stderr)
+  {
+    --let $line = $line 2> $output_file
+  }
+  if (!$redirect_stderr)
+  {
+    --let $line = $line > $output_file
+  }
+}
+
+--let $line = $line &
+
+if ($pid_file)
+{
+  --let $line = $line echo \$! > $pid_file
+}
+
+--let _LINE = $line
+--perl
+   my $cmd= $ENV{'_LINE'};
+   system("$cmd"); 
+EOF
+

mysql-test/suite/group_replication/r/gr_persist_only_variables.result

@@ -84,8 +84,10 @@ SET PERSIST_ONLY group_replication_tls_source = @@GLOBAL.group_replication_tls_s
 SET PERSIST_ONLY group_replication_transaction_size_limit = @@GLOBAL.group_replication_transaction_size_limit;
 SET PERSIST_ONLY group_replication_unreachable_majority_timeout = @@GLOBAL.group_replication_unreachable_majority_timeout;
 SET PERSIST_ONLY group_replication_view_change_uuid = @@GLOBAL.group_replication_view_change_uuid;
+SET PERSIST_ONLY group_replication_xcom_ssl_accept_retries = @@GLOBAL.group_replication_xcom_ssl_accept_retries;
+SET PERSIST_ONLY group_replication_xcom_ssl_socket_timeout = @@GLOBAL.group_replication_xcom_ssl_socket_timeout;
 
-include/assert.inc ['Expect 63 persisted variables.']
+include/assert.inc ['Expect 65 persisted variables.']
 
 ############################################################
 # 2. Restart server, it must bootstrap the group and preserve
@@ -94,9 +96,9 @@ include/assert.inc ['Expect 63 persisted variables.']
 include/rpl/reconnect.inc
 include/gr_wait_for_member_state.inc
 
-include/assert.inc ['Expect 63 persisted variables in persisted_variables table.']
-include/assert.inc ['Expect 62 variables which last value was set through SET PERSIST.']
-include/assert.inc ['Expect 52 persisted variables with matching persisted and global values.']
+include/assert.inc ['Expect 65 persisted variables in persisted_variables table.']
+include/assert.inc ['Expect 64 variables which last value was set through SET PERSIST.']
+include/assert.inc ['Expect 64 persisted variables with matching persisted and global values.']
 
 ############################################################
 # 3. Test RESET PERSIST IF EXISTS.
@@ -164,6 +166,8 @@ RESET PERSIST IF EXISTS group_replication_tls_source;
 RESET PERSIST IF EXISTS group_replication_transaction_size_limit;
 RESET PERSIST IF EXISTS group_replication_unreachable_majority_timeout;
 RESET PERSIST IF EXISTS group_replication_view_change_uuid;
+RESET PERSIST IF EXISTS group_replication_xcom_ssl_accept_retries;
+RESET PERSIST IF EXISTS group_replication_xcom_ssl_socket_timeout;
 
 include/assert.inc ['Expect 0 persisted variables.']
 

mysql-test/suite/group_replication/r/gr_persist_variables.result

@@ -88,8 +88,10 @@ SET PERSIST group_replication_unreachable_majority_timeout = @@GLOBAL.group_repl
 SET PERSIST group_replication_view_change_uuid = @@GLOBAL.group_replication_view_change_uuid;
 Warnings:
 Warning	1681	'group_replication_view_change_uuid' is deprecated and will be removed in a future release.
+SET PERSIST group_replication_xcom_ssl_accept_retries = @@GLOBAL.group_replication_xcom_ssl_accept_retries;
+SET PERSIST group_replication_xcom_ssl_socket_timeout = @@GLOBAL.group_replication_xcom_ssl_socket_timeout;
 
-include/assert.inc ['Expect 63 persisted variables.']
+include/assert.inc ['Expect 65 persisted variables.']
 
 ############################################################
 # 2. Restart server, it must bootstrap the group and preserve
@@ -98,9 +100,9 @@ include/assert.inc ['Expect 63 persisted variables.']
 include/rpl/reconnect.inc
 include/gr_wait_for_member_state.inc
 
-include/assert.inc ['Expect 63 persisted variables in persisted_variables table.']
-include/assert.inc ['Expect 62 variables which last value was set through SET PERSIST.']
-include/assert.inc ['Expect 62 variables which last value was set through SET PERSIST is equal to its global value.']
+include/assert.inc ['Expect 65 persisted variables in persisted_variables table.']
+include/assert.inc ['Expect 64 variables which last value was set through SET PERSIST.']
+include/assert.inc ['Expect 64 variables which last value was set through SET PERSIST is equal to its global value.']
 
 ############################################################
 # 3. Test RESET PERSIST.
@@ -168,6 +170,8 @@ RESET PERSIST group_replication_tls_source;
 RESET PERSIST group_replication_transaction_size_limit;
 RESET PERSIST group_replication_unreachable_majority_timeout;
 RESET PERSIST group_replication_view_change_uuid;
+RESET PERSIST group_replication_xcom_ssl_accept_retries;
+RESET PERSIST group_replication_xcom_ssl_socket_timeout;
 
 include/assert.inc ['Expect 0 persisted variables.']
 

mysql-test/suite/group_replication/r/gr_set_option_during_stop.result

@@ -45,6 +45,8 @@ WHERE VARIABLE_NAME LIKE 'group_replication_%'
  AND VARIABLE_NAME != 'group_replication_auto_evict_timeout'
  AND VARIABLE_NAME != 'group_replication_certification_loop_chunk_size'
  AND VARIABLE_NAME != 'group_replication_certification_loop_sleep_time'
+ AND VARIABLE_NAME != 'group_replication_xcom_ssl_socket_timeout'
+ AND VARIABLE_NAME != 'group_replication_xcom_ssl_accept_retries'
  ORDER BY VARIABLE_NAME;
 SET SESSION sql_log_bin = 1;
 SET @value= @@GLOBAL.group_replication_advertise_recovery_endpoints;
@@ -227,6 +229,10 @@ SET @value= @@GLOBAL.group_replication_tls_source;
 SET @@GLOBAL.group_replication_tls_source= @value;
 SET @value= @@GLOBAL.group_replication_transaction_size_limit;
 SET @@GLOBAL.group_replication_transaction_size_limit= @value;
+SET @value= @@GLOBAL.group_replication_xcom_ssl_accept_retries;
+SET @@GLOBAL.group_replication_xcom_ssl_accept_retries= @value;
+SET @value= @@GLOBAL.group_replication_xcom_ssl_socket_timeout;
+SET @@GLOBAL.group_replication_xcom_ssl_socket_timeout= @value;
 ############################################################
 # 5. Validate that we did test all Group Replication options.
 [connection server1]

mysql-test/suite/group_replication/r/gr_show_global_and_session_variables.result

@@ -7,8 +7,8 @@ Note	####	Storing MySQL user name or password information in the connection meta
 include/start_and_bootstrap_group_replication.inc
 include/stop_group_replication.inc
 
-# Test#1: Basic check that there are 64 GR variables.
-include/assert.inc [There are 64 GR variables at present.]
+# Test#1: Basic check that there are 66 GR variables.
+include/assert.inc [There are 66 GR variables at present.]
 
 # Test#2: Verify group replication related variables at GLOBAL scope.
 SET @@SESSION.group_replication_allow_local_lower_version_join= 1;

mysql-test/suite/group_replication/r/gr_ssl_socket_timeout.result

@@ -0,0 +1,50 @@
+include/group_replication.inc
+Warnings:
+Note	####	Sending passwords in plain text without SSL/TLS is extremely insecure.
+Note	####	Storing MySQL user name or password information in the connection metadata repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START REPLICA; see the 'START REPLICA Syntax' in the MySQL Manual for more information.
+[connection server1]
+
+############################################################
+# 1. Start one member with GCS SSL enabled.
+[connection server1]
+SET @group_replication_ssl_mode_save= @@GLOBAL.group_replication_ssl_mode;
+SET GLOBAL group_replication_ssl_mode= REQUIRED;
+SET @group_replication_xcom_ssl_socket_timeout_save= @@GLOBAL.group_replication_xcom_ssl_socket_timeout;
+SET @group_replication_xcom_ssl_accept_retries_save= @@GLOBAL.group_replication_xcom_ssl_accept_retries;
+SET GLOBAL group_replication_xcom_ssl_socket_timeout= 3;
+SET GLOBAL group_replication_xcom_ssl_accept_retries= 3;
+include/start_and_bootstrap_group_replication.inc
+Occurrences of 'Group communication SSL configuration: group_replication_ssl_mode: "REQUIRED"' in the input file: 1
+
+############################################################
+# 2. Start the second member with GCS SSL enabled, the member
+#    will be able to join the group.
+[connection server2]
+SET @group_replication_ssl_mode_save= @@GLOBAL.group_replication_ssl_mode;
+SET GLOBAL group_replication_ssl_mode= REQUIRED;
+include/start_group_replication.inc
+include/rpl/gr_wait_for_number_of_members.inc
+Occurrences of 'Group communication SSL configuration: group_replication_ssl_mode: "REQUIRED"' in the input file: 1
+
+############################################################
+# 3. Verify that any connection on group_replication
+#    communication port is aborted by the server after the
+#    timout configured by the group_replication_xcom_ssl_socket_timeout.
+include/stop_group_replication.inc
+SET @group_replication_communication_debug_options_save = @@GLOBAL.group_replication_communication_debug_options;
+SET GLOBAL group_replication_communication_debug_options= "XCOM_DEBUG_BASIC";
+START GROUP_REPLICATION;
+SET @@GLOBAL.group_replication_communication_debug_options= @group_replication_communication_debug_options_save;
+include/assert_grep.inc [Assert that the mysql connection has been ended by the server]
+include/assert_grep.inc [Assert that message about aborting the connection has been logged to GCS_DEBUG_TRACE file]
+ERROR 2013 (HY000): Lost connection to MySQL server at 'reading initial communication packet', system error: 2
+
+############################################################
+# 4. Clean up.
+[connection server1]
+SET GLOBAL group_replication_ssl_mode= @group_replication_ssl_mode_save;
+SET GLOBAL group_replication_xcom_ssl_socket_timeout= @group_replication_xcom_ssl_socket_timeout_save;
+SET GLOBAL group_replication_xcom_ssl_accept_retries= @group_replication_xcom_ssl_accept_retries_save;
+[connection server2]
+SET GLOBAL group_replication_ssl_mode= @group_replication_ssl_mode_save;
+include/group_replication_end.inc

mysql-test/suite/group_replication/r/gr_variables_default_values.result

@@ -28,9 +28,9 @@ include/stop_group_replication.inc
 #
 # Test Unit#1
 # Set global/session group replication variables to default.
-# Curently there are 64 group replication variables.
+# Curently there are 66 group replication variables.
 #
-include/assert.inc [There are 64 GR variables at present.]
+include/assert.inc [There are 66 GR variables at present.]
 SET @@GLOBAL.group_replication_auto_increment_increment= default;
 ERROR 42000: Variable 'group_replication_auto_increment_increment' can't be set to the value of 'DEFAULT'
 SET @@GLOBAL.group_replication_compression_threshold= default;
@@ -86,6 +86,8 @@ SET @@GLOBAL.group_replication_view_change_uuid= default;
 SET @@GLOBAL.group_replication_communication_stack = default;
 SET @@GLOBAL.group_replication_paxos_single_leader = default;
 SET @@GLOBAL.group_replication_preemptive_garbage_collection_rows_threshold = default;
+SET @@GLOBAL.group_replication_xcom_ssl_socket_timeout = default;
+SET @@GLOBAL.group_replication_xcom_ssl_accept_retries = default;
 SET @@SESSION.group_replication_consistency= default;
 #
 # Test Unit#2
@@ -137,6 +139,8 @@ include/assert.inc [Default group_replication_communication_stack is XCom]
 include/assert.inc [Default group_replication_paxos_single_leader is 0]
 include/assert.inc [Default group_replication_preemptive_garbage_collection is 0]
 include/assert.inc [Default group_replication_preemptive_garbage_collection_rows_threshold is 100000]
+include/assert.inc [Default group_replication_xcom_ssl_socket_timeout is 0]
+include/assert.inc [Default group_replication_xcom_ssl_accept_retries is 10]
 #
 # Clean up
 #

mysql-test/suite/group_replication/r/gr_variables_privileges.result

@@ -153,6 +153,10 @@ SET GLOBAL group_replication_unreachable_majority_timeout = @@GLOBAL.group_repli
 ERROR 42000: Access denied; you need (at least one of) the SUPER or SYSTEM_VARIABLES_ADMIN privilege(s) for this operation
 SET GLOBAL group_replication_view_change_uuid = @@GLOBAL.group_replication_view_change_uuid;
 ERROR 42000: Access denied; you need (at least one of) the SUPER or SYSTEM_VARIABLES_ADMIN privilege(s) for this operation
+SET GLOBAL group_replication_xcom_ssl_accept_retries = @@GLOBAL.group_replication_xcom_ssl_accept_retries;
+ERROR 42000: Access denied; you need (at least one of) the SUPER or SYSTEM_VARIABLES_ADMIN privilege(s) for this operation
+SET GLOBAL group_replication_xcom_ssl_socket_timeout = @@GLOBAL.group_replication_xcom_ssl_socket_timeout;
+ERROR 42000: Access denied; you need (at least one of) the SUPER or SYSTEM_VARIABLES_ADMIN privilege(s) for this operation
 
 # Like most system variables, setting the session value for
 # group_replication_consistency requires no special privileges.
@@ -236,6 +240,8 @@ SET GLOBAL group_replication_unreachable_majority_timeout = @@GLOBAL.group_repli
 SET GLOBAL group_replication_view_change_uuid = @@GLOBAL.group_replication_view_change_uuid;
 Warnings:
 Warning	1681	'group_replication_view_change_uuid' is deprecated and will be removed in a future release.
+SET GLOBAL group_replication_xcom_ssl_accept_retries = @@GLOBAL.group_replication_xcom_ssl_accept_retries;
+SET GLOBAL group_replication_xcom_ssl_socket_timeout = @@GLOBAL.group_replication_xcom_ssl_socket_timeout;
 
 ############################################################
 # 4. Grant GROUP_REPLICATION_ADMIN and verify setting

mysql-test/suite/group_replication/t/gr_set_option_during_stop.test

@@ -78,6 +78,8 @@ INSERT INTO gr_options_that_cannot_be_change (name)
  AND VARIABLE_NAME != 'group_replication_auto_evict_timeout'
  AND VARIABLE_NAME != 'group_replication_certification_loop_chunk_size'
  AND VARIABLE_NAME != 'group_replication_certification_loop_sleep_time'
+ AND VARIABLE_NAME != 'group_replication_xcom_ssl_socket_timeout'
+ AND VARIABLE_NAME != 'group_replication_xcom_ssl_accept_retries'
  ORDER BY VARIABLE_NAME;
 SET SESSION sql_log_bin = 1;
 --let $gr_options_that_cannot_be_change_count= `SELECT COUNT(*) FROM gr_options_that_cannot_be_change;`

mysql-test/suite/group_replication/t/gr_show_global_and_session_variables.test

@@ -30,7 +30,7 @@
 --source include/start_and_bootstrap_group_replication.inc
 --source include/stop_group_replication.inc
 
---let $gr_var_count= 64
+--let $gr_var_count= 66
 
 --echo
 --echo # Test#1: Basic check that there are $gr_var_count GR variables.