More tweaks to bulk tester. Added random seed option.

bulktest.php

@@ -32,8 +32,10 @@
 // Get the parameters from the URL.
 $contextid = required_param('contextid', PARAM_INT);
 $categoryid = optional_param('categoryid', null, PARAM_INT);
-$nruns = optional_param('nruns', 1, PARAM_INT);
+$randomseed = optional_param('randomseed', 0, PARAM_INT);
 $repeatrandomonly = optional_param('repeatrandomonly', 1, PARAM_INT);
+$nruns = optional_param('nruns', 1, PARAM_INT);
+
 
 // Login and check permissions.
 $context = context::instance_by_id($contextid);
@@ -62,7 +64,7 @@
 echo $OUTPUT->heading($title);
 
 // Run the tests.
-[$numpasses, $failingtests, $missinganswers] = $bulktester->run_all_tests_for_context($context, $categoryid, $repeatrandomonly, $nruns);
+[$numpasses, $failingtests, $missinganswers] = $bulktester->run_all_tests_for_context($context, $categoryid, $randomseed, $repeatrandomonly, $nruns);
 
 // Display the final summary.
 $bulktester->print_overall_result($numpasses, $failingtests, $missinganswers);

bulktestindex.php

@@ -84,7 +84,7 @@
 
         $testallurl = new moodle_url(
             '/question/type/coderunner/bulktest.php',
-            ['contextid' => $contextid, 'repeatrandomonly' => 1, 'nruns' => $nruns]
+            ['contextid' => $contextid, 'randomseed' => 0, 'repeatrandomonly' => 1, 'nruns' => $nruns]
         );
         $testalllink = html_writer::link(
             $testallurl,
@@ -120,7 +120,7 @@
             if ($cat->count > 0) {
                 $url = new moodle_url(
                     '/question/type/coderunner/bulktest.php',
-                    ['contextid' => $contextid, 'categoryid' => $cat->id, 'repeatrandomonly' => 1, 'nruns' => $nruns]
+                    ['contextid' => $contextid, 'categoryid' => $cat->id, 'randomseed' => 0, 'repeatrandomonly' => 1, 'nruns' => $nruns]
                 );
                 $linktext = $cat->name . ' (' . $cat->count . ')';
                 $link = html_writer::link($url, $linktext, ['style' => $buttonstyle]);

classes/bulk_tester.php

@@ -178,13 +178,19 @@ public function get_all_coderunner_questions_in_context($contextid, $includeprot
      *
      * @param context $context the context to run the tests for.
      * @param int $categoryid test only questions in this category. Default to all.
+     * @param int $randomseed used to set random seed before runs for each question. Default = 0 --- which means seed is not set.
+     *             Use this to have more chance of the series of questions being generated for testing is the same for a new run
+     *             of the tests. This works well with grader caching as you won't keep getting new random variations. Also allows
+     *             you to mix up the space that is being tested.
+     * @param int $repeatrandomonly when true(or 1), only repeats tests for questions with random in the name.
+     *              Default = true (or really 1).
      * @param int $nruns the number times to test each question. Default to 1.
      * @return array with three elements:
      *              int a count of how many tests passed
      *              array of messages relating to the questions with failures
      *              array of messages relating to the questions without sample answers
      */
-    public function run_all_tests_for_context(context $context, $categoryid = null, $repeatrandomonly = true, $nruns = 1) {
+    public function run_all_tests_for_context(context $context, $categoryid = null, $randomseed = 0, $repeatrandomonly = 1, $nruns = 1) {
         global $OUTPUT;
 
         // Load the necessary data.
@@ -234,7 +240,10 @@ public function run_all_tests_for_context(context $context, $categoryid = null,
                 } else {
                     $nrunsthistime = $nruns;
                 }
-                // Now run the test.
+                if ($randomseed > 0) {
+                    mt_srand($randomseed);
+                }
+                // Now run the test for the required number of times.
                 for ($i = 0; $i < $nrunsthistime; $i++) {
                     // only records last outcome and message
                     try {
@@ -261,9 +270,9 @@ public function run_all_tests_for_context(context $context, $categoryid = null,
 
                 // Report the result, and record failures for the summary.
                 if ($outcome != self::MISSINGANSWER) {
-                    echo "&nbsp;&nbsp;&nbsp;<i style='color:green;'>" . $passstr . "=" . $npasses. "</i>";
+                    echo "&nbsp;&nbsp;&nbsp;<i style='color:green;'>" . $passstr . "=" . $npasses . "</i>";
                     if ($nfails > 0) {
-                        echo ", <b style='color:red;'>" . $failstr . '=' . $nfails. "</b>";
+                        echo ", <b style='color:red;'>" . $failstr . '=' . $nfails . "</b>";
                     }
                 }
                 echo "</li>";
@@ -273,12 +282,12 @@ public function run_all_tests_for_context(context $context, $categoryid = null,
                 $qparams['qperpage'] = 1000;
                 $questionbankurl = new moodle_url('/question/edit.php', $qparams);
                 $questionbanklink = html_writer::link($questionbankurl, $nameandcount->name, ['target' => '_blank']);
-                if ($npasses == $nruns) {
-                    $numpasses += 1;
-                } else if ($outcome === self::MISSINGANSWER) {
+                if ($outcome === self::MISSINGANSWER) {
                     $missinganswers[] = "$coursename / $questionbanklink / $questionnamelink";
+                } else if ($nfails == 0) {
+                        $numpasses += 1;
                 } else {
-                    $failmessage = " <b style='color:red'>" . get_string('fail', 'qtype_coderunner') . '=' . $nfails. "</b>";
+                    $failmessage = " <b style='color:red'>" . get_string('fail', 'qtype_coderunner') . '=' . $nfails . "</b>";
                     $failingtests[] = "$coursename / $questionbanklink / $questionnamelink: $failmessage";
                 }
             }