PS-6849 Add "junit" test suite with deliberately failing tests

Adds number of tests that are failing in various ways to test how their failures are propagated to the JUnit report: 01_pass: always passes 02_disabled: passes, but is disabled by disabled.def 03_skipped: always skips itself 04_result_mismatch: result file contents differs 05_failed_assertion: asserts always-false condition 06_bootstrap_crash: crashes in the bootstrap phase 07_immediate_crash: crashes on command 08_shutdown_crash: crashes in the shutdown phase 09_boostrap_hang: waits indefinitely in the bootstrap phase 10_immediate_hang: waits indefinitely on command 11_shutdown_hang: waits indefinitely in the shutdown phase 12_shutdown_hang_in_master: master process of a master/slave pair waits indefinitely in the shutdown phase 13_shutdown_hang_in_slave: slave process of a master/slave pair waits indefinitely in the shutdown phase 14_shutdown_hang_in_both: both master and slave wait indefinitely in the shutdown phase 15_fail_every_second_time: fails every second time 16_bootstrap_buffer_overrun: writes behind a buffer during bootstrap 17_immediate_buffer_overrun: writes behind a buffer on command 18_shutdown_buffer_overrun: writes behind a buffer during shutdown 19_leak_memory: allocates but never frees a block of memory 20_leak_memory_in_master: master process leaks memory 21_leak_memory_in_slave: slave process leaks memory 22_leak_memory_in_both: both master and slave process leak memory
ldonoso · ldonoso · Jun 1, 2020 · Jun 1, 2020 · Jul 21, 2020 · Jul 27, 2020
commit a5d30eeb03a780b63cbc0f410c96f992c6605657
diff --git a/mysql-test/suite/junit/README b/mysql-test/suite/junit/README
@@ -0,0 +1,3 @@
+This suite contains a number of tests that intentionally fail in different
+ways. The tests are used to evaluate JUnit reporting capability of
+`mysql-test-run.pl`.
diff --git a/mysql-test/suite/junit/r/01_pass.result b/mysql-test/suite/junit/r/01_pass.result
@@ -0,0 +1 @@
+OK
diff --git a/mysql-test/suite/junit/r/02_disabled.result b/mysql-test/suite/junit/r/02_disabled.result
@@ -0,0 +1 @@
+OK
diff --git a/mysql-test/suite/junit/r/03_skipped.result b/mysql-test/suite/junit/r/03_skipped.result
@@ -0,0 +1 @@
+OK
diff --git a/mysql-test/suite/junit/r/04_result_mismatch.result b/mysql-test/suite/junit/r/04_result_mismatch.result
@@ -0,0 +1 @@
+OK
diff --git a/mysql-test/suite/junit/r/05_failed_assertion.result b/mysql-test/suite/junit/r/05_failed_assertion.result
@@ -0,0 +1 @@
+include/assert.inc ["0 = 1"]
diff --git a/mysql-test/suite/junit/r/06_bootstrap_crash.result b/mysql-test/suite/junit/r/06_bootstrap_crash.result
@@ -0,0 +1 @@
+OK
diff --git a/mysql-test/suite/junit/r/07_immediate_crash.result b/mysql-test/suite/junit/r/07_immediate_crash.result
@@ -0,0 +1 @@
+SET @@session.debug = "+d,crash_dispatch_command_before";
diff --git a/mysql-test/suite/junit/r/08_shutdown_crash.result b/mysql-test/suite/junit/r/08_shutdown_crash.result
@@ -0,0 +1 @@
+SET @@session.debug = "+d,shutdown_crash";
diff --git a/mysql-test/suite/junit/r/09_bootstrap_hang.result b/mysql-test/suite/junit/r/09_bootstrap_hang.result
@@ -0,0 +1 @@
+OK
diff --git a/mysql-test/suite/junit/r/10_immediate_hang.result b/mysql-test/suite/junit/r/10_immediate_hang.result
@@ -0,0 +1 @@
+SET @@session.debug = "+d,wait_forever";
diff --git a/mysql-test/suite/junit/r/11_shutdown_hang.result b/mysql-test/suite/junit/r/11_shutdown_hang.result
@@ -0,0 +1 @@
+SET @@session.debug = "+d,shutdown_hang";
diff --git a/mysql-test/suite/junit/r/12_shutdown_hang_in_master.result b/mysql-test/suite/junit/r/12_shutdown_hang_in_master.result
@@ -0,0 +1,7 @@
+include/master-slave.inc
+Warnings:
+Note	####	Sending passwords in plain text without SSL/TLS is extremely insecure.
+Note	####	Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
+[connection master]
+SET @@session.debug = "+d,shutdown_hang";
+include/rpl_end.inc
diff --git a/mysql-test/suite/junit/r/13_shutdown_hang_in_slave.result b/mysql-test/suite/junit/r/13_shutdown_hang_in_slave.result
@@ -0,0 +1,7 @@
+include/master-slave.inc
+Warnings:
+Note	####	Sending passwords in plain text without SSL/TLS is extremely insecure.
+Note	####	Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
+[connection master]
+SET @@session.debug = "+d,shutdown_hang";
+include/rpl_end.inc
diff --git a/mysql-test/suite/junit/r/14_shutdown_hang_in_both.result b/mysql-test/suite/junit/r/14_shutdown_hang_in_both.result
@@ -0,0 +1,8 @@
+include/master-slave.inc
+Warnings:
+Note	####	Sending passwords in plain text without SSL/TLS is extremely insecure.
+Note	####	Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
+[connection master]
+SET @@session.debug = "+d,shutdown_hang";
+SET @@session.debug = "+d,shutdown_hang";
+include/rpl_end.inc
diff --git a/mysql-test/suite/junit/r/15_fail_every_second_time.result b/mysql-test/suite/junit/r/15_fail_every_second_time.result
@@ -0,0 +1 @@
+tick
diff --git a/mysql-test/suite/junit/r/16_bootstrap_buffer_overrun.result b/mysql-test/suite/junit/r/16_bootstrap_buffer_overrun.result
@@ -0,0 +1 @@
+OK
diff --git a/mysql-test/suite/junit/r/17_immediate_buffer_overrun.result b/mysql-test/suite/junit/r/17_immediate_buffer_overrun.result
@@ -0,0 +1 @@
+SET @@session.debug = "+d,buffer_overrun";
diff --git a/mysql-test/suite/junit/r/18_shutdown_buffer_overrun.result b/mysql-test/suite/junit/r/18_shutdown_buffer_overrun.result
@@ -0,0 +1 @@
+SET @@session.debug = "+d,shutdown_buffer_overrun";
diff --git a/mysql-test/suite/junit/r/19_leak_memory.result b/mysql-test/suite/junit/r/19_leak_memory.result
@@ -0,0 +1 @@
+SET @@session.debug = "+d,leak_memory";
diff --git a/mysql-test/suite/junit/r/20_leak_memory_in_master.result b/mysql-test/suite/junit/r/20_leak_memory_in_master.result
@@ -0,0 +1,7 @@
+include/master-slave.inc
+Warnings:
+Note	####	Sending passwords in plain text without SSL/TLS is extremely insecure.
+Note	####	Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
+[connection master]
+SET @@session.debug = "+d,leak_memory";
+include/rpl_end.inc
diff --git a/mysql-test/suite/junit/r/21_leak_memory_in_slave.result b/mysql-test/suite/junit/r/21_leak_memory_in_slave.result
@@ -0,0 +1,7 @@
+include/master-slave.inc
+Warnings:
+Note	####	Sending passwords in plain text without SSL/TLS is extremely insecure.
+Note	####	Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
+[connection master]
+SET @@session.debug = "+d,leak_memory";
+include/rpl_end.inc
diff --git a/mysql-test/suite/junit/r/22_leak_memory_in_both.result b/mysql-test/suite/junit/r/22_leak_memory_in_both.result
@@ -0,0 +1,8 @@
+include/master-slave.inc
+Warnings:
+Note	####	Sending passwords in plain text without SSL/TLS is extremely insecure.
+Note	####	Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
+[connection master]
+SET @@session.debug = "+d,leak_memory";
+SET @@session.debug = "+d,leak_memory";
+include/rpl_end.inc
diff --git a/mysql-test/suite/junit/t/01_pass.test b/mysql-test/suite/junit/t/01_pass.test
@@ -0,0 +1,2 @@
+# Empty test. Always passes.
+--echo OK
diff --git a/mysql-test/suite/junit/t/02_disabled.test b/mysql-test/suite/junit/t/02_disabled.test
@@ -0,0 +1,2 @@
+# Disabled test. See `disabled.def`.
+--echo OK
diff --git a/mysql-test/suite/junit/t/03_skipped.test b/mysql-test/suite/junit/t/03_skipped.test
@@ -0,0 +1,3 @@
+# Test that skips itself.
+--skip expected to be skipped
+--echo OK
diff --git a/mysql-test/suite/junit/t/04_result_mismatch.test b/mysql-test/suite/junit/t/04_result_mismatch.test
@@ -0,0 +1,4 @@
+# Result mismatch.
+
+# '.result' file expects "OK".
+--echo Not OK
diff --git a/mysql-test/suite/junit/t/05_failed_assertion.test b/mysql-test/suite/junit/t/05_failed_assertion.test
@@ -0,0 +1,5 @@
+# Failed assertion.
+
+--let $assert_text = "0 = 1"
+--let $assert_cond = 0 = 1
+--source include/assert.inc
diff --git a/mysql-test/suite/junit/t/06_bootstrap_crash-master.opt b/mysql-test/suite/junit/t/06_bootstrap_crash-master.opt
@@ -0,0 +1 @@
+--debug="+d,bootstrap_crash" --init-file=$MYSQL_TEST_DIR/suite/junit/t/select_1.sql
diff --git a/mysql-test/suite/junit/t/06_bootstrap_crash.test b/mysql-test/suite/junit/t/06_bootstrap_crash.test
@@ -0,0 +1,4 @@
+# Crashing in the bootstrap stage. See '.opt' file.
+--source include/have_debug.inc
+
+--echo OK
diff --git a/mysql-test/suite/junit/t/07_immediate_crash.test b/mysql-test/suite/junit/t/07_immediate_crash.test
@@ -0,0 +1,5 @@
+# Crashes almost immediately.
+
+--source include/have_debug.inc
+
+SET @@session.debug = "+d,crash_dispatch_command_before";
diff --git a/mysql-test/suite/junit/t/08_shutdown_crash.test b/mysql-test/suite/junit/t/08_shutdown_crash.test
@@ -0,0 +1,5 @@
+# Crashing during shutdown.
+
+--source include/have_debug.inc
+
+SET @@session.debug = "+d,shutdown_crash";
diff --git a/mysql-test/suite/junit/t/09_bootstrap_hang-master.opt b/mysql-test/suite/junit/t/09_bootstrap_hang-master.opt
@@ -0,0 +1 @@
+--debug="+d,bootstrap_hang" --init-file=$MYSQL_TEST_DIR/suite/junit/t/select_1.sql
diff --git a/mysql-test/suite/junit/t/09_bootstrap_hang.test b/mysql-test/suite/junit/t/09_bootstrap_hang.test
@@ -0,0 +1,4 @@
+# Waiting indefinitely in the bootstrap stage. See '.opt' file.
+--source include/have_debug.inc
+
+--echo OK
diff --git a/mysql-test/suite/junit/t/10_immediate_hang.test b/mysql-test/suite/junit/t/10_immediate_hang.test
@@ -0,0 +1,5 @@
+# Hangs immediately.
+
+--source include/have_debug.inc
+
+SET @@session.debug = "+d,wait_forever";
diff --git a/mysql-test/suite/junit/t/11_shutdown_hang.test b/mysql-test/suite/junit/t/11_shutdown_hang.test
@@ -0,0 +1,5 @@
+# Hangs during shutdown.
+
+--source include/have_debug.inc
+
+SET @@session.debug = "+d,shutdown_hang";
diff --git a/mysql-test/suite/junit/t/12_shutdown_hang_in_master.test b/mysql-test/suite/junit/t/12_shutdown_hang_in_master.test
@@ -0,0 +1,9 @@
+# Hangs master process during shutdown.
+
+--source include/have_debug.inc
+--source include/master-slave.inc
+
+--connection master
+SET @@session.debug = "+d,shutdown_hang";
+
+--source include/rpl_end.inc
diff --git a/mysql-test/suite/junit/t/13_shutdown_hang_in_slave.test b/mysql-test/suite/junit/t/13_shutdown_hang_in_slave.test
@@ -0,0 +1,9 @@
+# Hangs slave process during shutdown.
+
+--source include/have_debug.inc
+--source include/master-slave.inc
+
+--connection slave
+SET @@session.debug = "+d,shutdown_hang";
+
+--source include/rpl_end.inc
diff --git a/mysql-test/suite/junit/t/14_shutdown_hang_in_both.test b/mysql-test/suite/junit/t/14_shutdown_hang_in_both.test
@@ -0,0 +1,12 @@
+# Hangs both master and slave processes during shutdown.
+
+--source include/have_debug.inc
+--source include/master-slave.inc
+
+--connection master
+SET @@session.debug = "+d,shutdown_hang";
+
+--connection slave
+SET @@session.debug = "+d,shutdown_hang";
+
+--source include/rpl_end.inc
diff --git a/mysql-test/suite/junit/t/15_fail_every_second_time.test b/mysql-test/suite/junit/t/15_fail_every_second_time.test
@@ -0,0 +1,26 @@
+# Fails every second time.
+
+--source include/have_debug.inc
+
+--perl
+use strict;
+use warnings;
+
+my $fname = "$ENV{MYSQLTEST_VARDIR}/junit-15_fail_every_second_time-state.txt";
+my $newval = "tick";
+
+if (open my $f, "<", $fname) {
+    if (defined(my $val = <$f>)) {
+        chomp $val;
+        close $f;
+        $newval = $val eq "tock" ? "tick" : "tock";
+    }
+}
+
+open my $f2, ">", $fname or die("open>");
+print $f2 "$newval\n";
+close $f2;
+
+EOF
+
+--cat_file $MYSQLTEST_VARDIR/junit-15_fail_every_second_time-state.txt
diff --git a/mysql-test/suite/junit/t/16_bootstrap_buffer_overrun-master.opt b/mysql-test/suite/junit/t/16_bootstrap_buffer_overrun-master.opt
@@ -0,0 +1 @@
+--debug="+d,bootstrap_buffer_overrun" --init-file=$MYSQL_TEST_DIR/suite/junit/t/select_1.sql
diff --git a/mysql-test/suite/junit/t/16_bootstrap_buffer_overrun.test b/mysql-test/suite/junit/t/16_bootstrap_buffer_overrun.test
@@ -0,0 +1,4 @@
+# Buffer overrun in the bootstrap stage. See '.opt' file.
+--source include/have_debug.inc
+
+--echo OK
diff --git a/mysql-test/suite/junit/t/17_immediate_buffer_overrun.test b/mysql-test/suite/junit/t/17_immediate_buffer_overrun.test
@@ -0,0 +1,5 @@
+# Causes buffer overrun, as soon as possible.
+
+--source include/have_debug.inc
+
+SET @@session.debug = "+d,buffer_overrun";
diff --git a/mysql-test/suite/junit/t/18_shutdown_buffer_overrun.test b/mysql-test/suite/junit/t/18_shutdown_buffer_overrun.test
@@ -0,0 +1,5 @@
+# Causes buffer overrun during shutdown.
+
+--source include/have_debug.inc
+
+SET @@session.debug = "+d,shutdown_buffer_overrun";
diff --git a/mysql-test/suite/junit/t/19_leak_memory.test b/mysql-test/suite/junit/t/19_leak_memory.test
@@ -0,0 +1,5 @@
+# Leaks memory.
+
+--source include/have_debug.inc
+
+SET @@session.debug = "+d,leak_memory";
diff --git a/mysql-test/suite/junit/t/20_leak_memory_in_master.test b/mysql-test/suite/junit/t/20_leak_memory_in_master.test
@@ -0,0 +1,9 @@
+# Leaks memory in master process.
+
+--source include/have_debug.inc
+--source include/master-slave.inc
+
+--connection master
+SET @@session.debug = "+d,leak_memory";
+
+--source include/rpl_end.inc
diff --git a/mysql-test/suite/junit/t/21_leak_memory_in_slave.test b/mysql-test/suite/junit/t/21_leak_memory_in_slave.test
@@ -0,0 +1,9 @@
+# Leaks memory in slave process.
+
+--source include/have_debug.inc
+--source include/master-slave.inc
+
+--connection slave
+SET @@session.debug = "+d,leak_memory";
+
+--source include/rpl_end.inc
diff --git a/mysql-test/suite/junit/t/22_leak_memory_in_both.test b/mysql-test/suite/junit/t/22_leak_memory_in_both.test
@@ -0,0 +1,12 @@
+# Leaks memory in both master and slave processes.
+
+--source include/have_debug.inc
+--source include/master-slave.inc
+
+--connection master
+SET @@session.debug = "+d,leak_memory";
+
+--connection slave
+SET @@session.debug = "+d,leak_memory";
+
+--source include/rpl_end.inc
diff --git a/mysql-test/suite/junit/t/disabled.def b/mysql-test/suite/junit/t/disabled.def
@@ -0,0 +1 @@
+02_disabled: expected to be disabled
diff --git a/mysql-test/suite/junit/t/select_1.sql b/mysql-test/suite/junit/t/select_1.sql
@@ -0,0 +1 @@
+SELECT 1;
diff --git a/sql/mysqld.cc b/sql/mysqld.cc
@@ -6415,6 +6415,20 @@ static void bootstrap(MYSQL_FILE *file)
   thd->thread_id= thd->variables.pseudo_thread_id= thread_id++;
 
   in_bootstrap= TRUE;
+  DBUG_EXECUTE_IF("bootstrap_crash", DBUG_SUICIDE(););
+  DBUG_EXECUTE_IF("bootstrap_hang", {
+    while (1)
+      my_sleep(1000000);
+  });
+  DBUG_EXECUTE_IF("bootstrap_buffer_overrun", {
+    int *mem = static_cast<int *>(my_malloc(127, 0));
+    // Allocations are usually aligned, so even if 127 bytes were requested,
+    // it's mostly safe to assume there are 128 bytes. Writing into the last
+    // byte is safe for the rest of the code, but still enough to trigger
+    // AddressSanitizer (ASAN) or Valgrind.
+    my_atomic_store32(mem + (128 / sizeof(*mem)) - 1, 1);
+    free(mem);
+  });
 
   bootstrap_file=file;
 #ifndef EMBEDDED_LIBRARY      // TODO:  Enable this

diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc
@@ -5421,3 +5421,6 @@ static Sys_var_mybool Sys_show_old_temporals(
         ON_CHECK(0), ON_UPDATE(0),
         DEPRECATED(""));
 
+#ifndef DBUG_OFF
+Debug_shutdown_actions Debug_shutdown_actions::instance;
+#endif
diff --git a/sql/sys_vars.h b/sql/sys_vars.h
@@ -704,6 +704,46 @@ class Sys_var_lexstring: public Sys_var_charptr
 };
 
 #ifndef DBUG_OFF
+
+static inline void trigger_buffer_overrun()
+{
+  int *mem = static_cast<int *>(my_malloc(127, 0));
+  // Allocations are usually aligned, so even if 127 bytes were requested,
+  // it's mostly safe to assume there are 128 bytes. Writing into the last
+  // byte is safe for the rest of the code, but still enough to trigger
+  // AddressSanitizer (ASAN) or Valgrind.
+  my_atomic_store32(mem + (128 / sizeof(*mem)) - 1, 1);
+  free(mem);
+}
+
+class Debug_shutdown_actions
+{
+public:
+  static Debug_shutdown_actions instance;
+
+  bool crash;
+  bool hang;
+  bool buffer_overrun;
+
+public:
+  Debug_shutdown_actions()
+    : crash(false), hang(false), buffer_overrun(false)
+  {}
+
+  ~Debug_shutdown_actions()
+  {
+    if (crash)
+      DBUG_SUICIDE();
+
+    if (hang)
+      while (1)
+        my_sleep(1000000);
+
+    if (buffer_overrun)
+      trigger_buffer_overrun();
+  }
+};
+
 /**
   @@session.dbug and @@global.dbug variables.
 
@@ -750,6 +790,26 @@ class Sys_var_dbug: public sys_var
       DBUG_POP();
     else
       DBUG_SET(val);
+
+    DBUG_EXECUTE_IF("shutdown_crash", {
+      Debug_shutdown_actions::instance.crash = true;
+    });
+    DBUG_EXECUTE_IF("shutdown_hang", {
+      Debug_shutdown_actions::instance.hang = true;
+    });
+    DBUG_EXECUTE_IF("shutdown_buffer_overrun", {
+      Debug_shutdown_actions::instance.buffer_overrun = true;
+    });
+    DBUG_EXECUTE_IF("leak_memory", {
+      my_malloc(127, 0); // Intentional memory leak.
+    });
+    DBUG_EXECUTE_IF("wait_forever", {
+      while (1)
+        my_sleep(1000000);
+    });
+    DBUG_EXECUTE_IF("buffer_overrun", {
+      trigger_buffer_overrun();
+    });
     return false;
   }
   bool global_update(THD *thd, set_var *var)
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		SET @@session.debug = "+d,crash_dispatch_command_before";
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		SET @@session.debug = "+d,shutdown_buffer_overrun";
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		# Disabled test. See `disabled.def`.
		--echo OK
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		--debug="+d,bootstrap_crash" --init-file=$MYSQL_TEST_DIR/suite/junit/t/select_1.sql
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		--debug="+d,bootstrap_hang" --init-file=$MYSQL_TEST_DIR/suite/junit/t/select_1.sql
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		--debug="+d,bootstrap_buffer_overrun" --init-file=$MYSQL_TEST_DIR/suite/junit/t/select_1.sql