Skip to content

Commit

Permalink
Updated behat tests to initialise jobe sandbox properly each time
Browse files Browse the repository at this point in the history
May still need to change behat_coderunner.php to make it more general.
That is make it possible to change the jobeserver address in different
situations, eg, when running in docker locally or docker in CI tests
on github.
  • Loading branch information
mckeownp committed Mar 3, 2024
1 parent e0f7ac9 commit feb3b5d
Show file tree
Hide file tree
Showing 34 changed files with 155 additions and 86 deletions.
102 changes: 60 additions & 42 deletions classes/jobesandbox.php
Original file line number Diff line number Diff line change
Expand Up @@ -75,12 +75,27 @@ public function __construct() {
qtype_coderunner_sandbox::__construct();

// Hack to force use of a local jobe host when behat testing.
if ($CFG->prefix == "bht_") {
$this->jobeserver = "localhost";
} else {
$this->jobeserver = get_config('qtype_coderunner', 'jobe_host');
// if ($CFG->prefix == "bht_") {
// $this->jobeserver = "localhost"; // Should it be :4000 ?
// } else if ($CFG->prefix == "b_") {
// $this->jobeserver = "172.17.0.1:4000"; // For local moodle-docker usage.
// } else {
// $this->jobeserver = get_config('qtype_coderunner', 'jobe_host');
// }

// use Given the CodeRunner jobe sandbox is enabled
// or Given the CodeRunner jobe scratchpad is enabled
// in each feature to make sure the right jobe_host string is used
// adjust behat_coderunner.php to set the appropriate server
// eg, in a local docker we usually have 172.17.0.1:4000

$jobefromconfig = get_config('qtype_coderunner', 'jobe_host');
$this->jobeserver = $jobefromconfig;
$keyenabled = get_config('qtype_coderunner', 'jobe_apikey_enabled');
// $this->jobeserver = $jobefromconfig;
if ($jobefromconfig != "172.17.0.1:4000") {
$banana = 'what?';
}

$this->apikey = get_config('qtype_coderunner', 'jobe_apikey');
$this->languages = null;
}
Expand Down Expand Up @@ -220,10 +235,10 @@ public function execute($sourcecode, $language, $input, $files = null, $params =
// But, remember that the server is chosen at random from the pool!

$key = hash("md5", serialize($runspec));
// echo '<pre>' . serialize($runspec) . '</pre>';
$runresult = $cache->get($key); // unersializes the returned value :) false if not found.
// Debugger: echo '<pre>' . serialize($runspec) . '</pre>';.
$runresult = $cache->get($key); // Unserializes the returned value :) false if not found.
if ($runresult) {
// echo $key . '-----------> FOUND' . '<br>';
// echo $key . '-----------> FOUND' . '<br>'; .
}
}

Expand Down Expand Up @@ -277,10 +292,10 @@ public function execute($sourcecode, $language, $input, $files = null, $params =
$runresult['cmpinfo'] = $this->response->cmpinfo;
$runresult['output'] = $this->filter_file_path($this->response->stdout);

// Got a useable result from Jobe server so cache it if required
// Got a useable result from Jobe server so cache it if required.
if (WRITE_TO_CACHE) {
$key = hash("md5", serialize($runspec));
$cache->set($key, $runresult); // set serializes the result, get will unserialize
$cache->set($key, $runresult); // set serializes the result, get will unserialize.
// echo 'CACHE WRITE for ---> ' . $key . '<br>';
}
}
Expand All @@ -292,58 +307,61 @@ public function execute($sourcecode, $language, $input, $files = null, $params =
// such class found. Removes comments, strings and nested code and then
// uses a regexp to find a public class.
private function get_main_class($prog) {
// filter out comments and strings
// Filter out comments and strings.
$prog = $prog . ' ';
$filteredProg = array();
$skipTo = -1;
$filteredprog = [];
$skipto = -1;

for ($i = 0; $i < strlen($prog) - 1; $i++) {
if ($skipTo == false) break; // an unclosed comment/string - bail out
if ($i < $skipTo) continue;

// skip "//" comments
if ($prog[$i].$prog[$i+1] == '//') {
$skipTo = strpos($prog, "\n", $i + 2);
if ($skipto == false) {
break; // An unclosed comment/string - bail out.
}

// skip "/**/" comments
else if ($prog[$i].$prog[$i+1] == '/*') {
$skipTo = strpos($prog, '*/', $i + 2) + 2;
$filteredProg[] = ' '; // '/**/' is a token delimiter
if ($i < $skipto) {
continue;
}

// skip strings
else if ($prog[$i] == '"') {
// matches the whole string
// Skip "//" comments.
if ($prog[$i] . $prog[$i + 1] == '//') {
$skipto = strpos($prog, "\n", $i + 2);
// Skip "/**/" comments.
} else if ($prog[$i] . $prog[$i + 1] == '/*') {
$skipto = strpos($prog, '*/', $i + 2) + 2;
$filteredprog[] = ' '; // The string '/**/' is a token delimiter.
// Skip strings.
} else if ($prog[$i] == '"') {
// Matches the whole string.
if (preg_match('/"((\\.)|[^\\"])*"/', $prog, $matches, 0, $i)) {
$skipTo = $i + strlen($matches[0]);
$skipto = $i + strlen($matches[0]);
} else {
$skipto = false;
}
else $skipTo = false;
// Copy everything else.
} else {
$filteredprog[] = $prog[$i];
}

// copy everything else
else $filteredProg[] = $prog[$i];
}

// remove nested code
// Remove nested code.
$depth = 0;
for ($i = 0; $i < count($filteredProg); $i++) {
if ($filteredProg[$i] == '{') $depth++;
if ($filteredProg[$i] == '}') $depth--;
if ($filteredProg[$i] != "\n" && $depth > 0 && !($depth == 1 && $filteredProg[$i] == '{')) {
$filteredProg[$i] = ' ';
for ($i = 0; $i < count($filteredprog); $i++) {
if ($filteredprog[$i] == '{') {
$depth++;
}
if ($filteredprog[$i] == '}') {
$depth--;
}
if ($filteredprog[$i] != "\n" && $depth > 0 && !($depth == 1 && $filteredprog[$i] == '{')) {
$filteredprog[$i] = ' ';
}
}

// search for a public class
if (preg_match('/public\s(\w*\s)*class\s*(\w+)[^\w]/', implode('', $filteredProg), $matches) !== 1) {
// Search for a public class.
if (preg_match('/public\s(\w*\s)*class\s*(\w+)[^\w]/', implode('', $filteredprog), $matches) !== 1) {
return false;
} else {
return $matches[2];
}
}



// Return the sandbox error code corresponding to the given httpcode.
private function get_error_code($httpcode) {
Expand Down
2 changes: 0 additions & 2 deletions question.php
Original file line number Diff line number Diff line change
Expand Up @@ -843,8 +843,6 @@ public function display_feedback() {
* the history of prior submissions.
* @param bool $isprecheck true iff this grading is occurring because the
* student clicked the precheck button
* @param bool $usecache If true (and the coderunner cachegradingresults setting is also true) then
* cache results in coderunner cache and use results from the coderunner grading cache.
* @return 3-element array of the mark (0 - 1), the question_state (
* gradedright, gradedwrong, gradedpartial, invalid) and the full
* qtype_coderunner_testing_outcome object to be cached. The invalid
Expand Down
4 changes: 2 additions & 2 deletions tests/behat/ace_scratchpad_compatibility.feature
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ Feature: Ace UI convert to Scratchpad UI questions with one click
I should be able to change a question from using Ace to Scratchpad in one click

Background:
Given the following "users" exist:
Given the CodeRunner scratchpad is enabled
And the following "users" exist:
| username | firstname | lastname | email |
| teacher1 | Teacher | 1 | teacher1@asd.com |
And the following "courses" exist:
Expand All @@ -20,7 +21,6 @@ Feature: Ace UI convert to Scratchpad UI questions with one click
And the following "questions" exist:
| questioncategory | qtype | name |
| Test questions | coderunner | Square function |
And the CodeRunner sandbox is enabled

When I am on the "Square function" "core_question > edit" page logged in as teacher1
And I set the following fields to these values:
Expand Down
4 changes: 2 additions & 2 deletions tests/behat/attachmentimportexport.feature
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ Feature: Test importing and exporting of question with attachments
I need to be able to import and export them

Background:
Given the following "users" exist:
Given the CodeRunner jobe sandbox is enabled
And the following "users" exist:
| username |
| teacher |
And the following "courses" exist:
Expand All @@ -20,7 +21,6 @@ Feature: Test importing and exporting of question with attachments
And the following "questions" exist:
| questioncategory | qtype | name |
| Test questions | coderunner | Square function |
And the CodeRunner sandbox is enabled
And I am on the "Square function" "core_question > edit" page logged in as teacher
And I click on "a[aria-controls='id_attachmentoptionscontainer']" "css_element"
And I set the field "Answer" to "from sqrmodule import sqr"
Expand Down
3 changes: 2 additions & 1 deletion tests/behat/attachments.feature
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ Feature: Test editing and using attachments to a CodeRunner question
I need to enable and configure them, then preview them.

Background:
Given the following "users" exist:
Given the CodeRunner jobe sandbox is enabled
And the following "users" exist:
| username | firstname | lastname | email |
| teacher1 | Teacher | 1 | teacher1@asd.com |
And the following "courses" exist:
Expand Down
1 change: 1 addition & 0 deletions tests/behat/backup_and_restore.feature
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ Feature: Duplicate a course containing a CodeRunner question
I need to be able to back them up and restore them

Background:
Given the CodeRunner jobe sandbox is enabled
And the following "courses" exist:
| fullname | shortname | category |
| Course 1 | C1 | 0 |
Expand Down
26 changes: 25 additions & 1 deletion tests/behat/behat_coderunner.php
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,38 @@ class behat_coderunner extends behat_base {
/**
* Sets the webserver sandbox to enabled for testing purposes.
*
* @Given /^the CodeRunner sandbox is enabled/
* @Given /^the CodeRunner jobe sandbox is enabled/
*/
public function the_coderunner_sandbox_is_enabled() {
set_config('jobesandbox_enabled', 1, 'qtype_coderunner');
set_config('jobe_host', '172.17.0.1:4000', 'qtype_coderunner');
}


/**
* Sets the webserver scratchpad to enabled for testing purposes.
*
* @Given /^the CodeRunner scratchpad is enabled/
*/
public function the_coderunner_scratchpad_is_enabled() {
set_config('wsenabled', 1, 'qtype_coderunner');
set_config('jobesandbox_enabled', 1, 'qtype_coderunner');
set_config('jobe_host', '172.17.0.1:4000', 'qtype_coderunner');
}


/**
* Sets the webserver scratchpad to disabled for testing purposes.
*
* @Given /^the CodeRunner scratchpad is disabled/
*/
public function the_coderunner_scratchpad_is_disabled() {
set_config('wsenabled', 0, 'qtype_coderunner');
set_config('jobesandbox_enabled', 1, 'qtype_coderunner');
set_config('jobe_host', '172.17.0.1:4000', 'qtype_coderunner');
}


/**
* Checks that a given string appears within answer textarea.
* Intended for checking UI serialization
Expand Down
3 changes: 2 additions & 1 deletion tests/behat/check_graph_question_types.feature
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ Feature: Check that the directed and undirected graph question types work.
I should be able to write simple graph questions and have them work correctly

Background:
Given the following "users" exist:
Given the CodeRunner jobe sandbox is enabled
And the following "users" exist:
| username | firstname | lastname | email |
| teacher1 | Teacher | 1 | teacher1@asd.com |
And the following "courses" exist:
Expand Down
3 changes: 2 additions & 1 deletion tests/behat/check_python_template_params.feature
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ Feature: Check that Python and other languages can be used instead of Twig as a
I should be able to write a function that prints the seed and my username it should be marked right

Background:
Given the following "users" exist:
Given the CodeRunner jobe sandbox is enabled
And the following "users" exist:
| username | firstname | lastname | email |
| teacher1 | Teacher | Last | teacher1@asd.com |
| student1 | Student First | O'Connell | student@asd.com |
Expand Down
3 changes: 2 additions & 1 deletion tests/behat/check_stepinfo.feature
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ Feature: Check that the QUESTION.stepinfo record is working.
I should be able to write a question that gives different feedback for different submissions.

Background:
Given the following "users" exist:
Given the CodeRunner jobe sandbox is enabled
And the following "users" exist:
| username | firstname | lastname | email |
| teacher1 | Teacher | 1 | teacher1@asd.com |
| student1 | Student | 1 | student@asd.com |
Expand Down
3 changes: 2 additions & 1 deletion tests/behat/check_twig_student_variable.feature
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ Feature: Check the STUDENT Twig variable allows access to current username in Co
I should be able to write a function that prints my username it should be marked right

Background:
Given the following "users" exist:
Given the CodeRunner jobe sandbox is enabled
And the following "users" exist:
| username | firstname | lastname | email |
| teacher1 | Teacher | 1 | teacher1@asd.com |
| student1 | Student | 1 | student@asd.com |
Expand Down
3 changes: 2 additions & 1 deletion tests/behat/create_python3_sqr_function.feature
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ Feature: Create a CodeRunner question (the sqr function example)
I need to create a new CodeRunner question

Background:
Given the following "users" exist:
Given the CodeRunner jobe sandbox is enabled
And the following "users" exist:
| username | firstname | lastname | email |
| teacher1 | Teacher | 1 | teacher1@asd.com |
And the following "courses" exist:
Expand Down
3 changes: 2 additions & 1 deletion tests/behat/duplicate_prototype.feature
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ Feature: duplicate_prototypes
I should see an informative error message and be able to fix by editing the duplicates

Background:
Given the following "users" exist:
Given the CodeRunner jobe sandbox is enabled
And the following "users" exist:
| username | firstname | lastname | email |
| teacher1 | Teacher | 1 | teacher1@asd.com |
And the following "courses" exist:
Expand Down
3 changes: 2 additions & 1 deletion tests/behat/edit.feature
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ Feature: Test editing a CodeRunner question
I need to edit them

Background:
Given the following "users" exist:
Given the CodeRunner jobe sandbox is enabled
And the following "users" exist:
| username | firstname | lastname | email |
| teacher1 | Teacher | 1 | teacher1@asd.com |
And the following "courses" exist:
Expand Down
3 changes: 2 additions & 1 deletion tests/behat/edit_question_precheck.feature
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ Feature: edit_question_precheck
I should get informative error messages if saving was unsuccessful

Background:
Given the following "users" exist:
Given the CodeRunner jobe sandbox is enabled
And the following "users" exist:
| username | firstname | lastname | email |
| teacher1 | Teacher | 1 | teacher1@asd.com |
And the following "courses" exist:
Expand Down
3 changes: 2 additions & 1 deletion tests/behat/edit_table.feature
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ Feature: Test editing a CodeRunner question using the Table UI
I should be able to set the table headers and see the table in the edit form.

Background:
Given the following "users" exist:
Given the CodeRunner jobe sandbox is enabled
And the following "users" exist:
| username | firstname | lastname | email |
| teacher1 | Teacher | 1 | teacher1@asd.com |
And the following "courses" exist:
Expand Down
3 changes: 2 additions & 1 deletion tests/behat/export.feature
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ Feature: Export CodeRunner questions
I need to export them

Background:
Given the following "users" exist:
Given the CodeRunner jobe sandbox is enabled
And the following "users" exist:
| username |
| teacher |
And the following "courses" exist:
Expand Down
3 changes: 2 additions & 1 deletion tests/behat/gapfiller_ui.feature
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ Feature: Test the GapFiller_UI
I should be able specify the required gaps in the global extra or test0 fields

Background:
Given the following "users" exist:
Given the CodeRunner jobe sandbox is enabled
And the following "users" exist:
| username | firstname | lastname | email |
| teacher1 | Teacher | 1 | teacher1@asd.com |
And the following "courses" exist:
Expand Down
3 changes: 2 additions & 1 deletion tests/behat/grading_scenarios.feature
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ Feature: Check grading with the Python 3 sqr function CodeRunner question
I must be able to preview them

Background:
Given the following "users" exist:
Given the CodeRunner jobe sandbox is enabled
And the following "users" exist:
| username | firstname | lastname | email |
| teacher1 | Teacher | 1 | teacher1@asd.com |
And the following "courses" exist:
Expand Down
3 changes: 2 additions & 1 deletion tests/behat/html_ui.feature
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ Feature: Test the HTML_UI
I should be able specify the required html in either globalextra or prototypeextra

Background:
Given the following "users" exist:
Given the CodeRunner jobe sandbox is enabled
And the following "users" exist:
| username | firstname | lastname | email |
| teacher1 | Teacher | 1 | teacher1@asd.com |
And the following "courses" exist:
Expand Down
3 changes: 2 additions & 1 deletion tests/behat/import.feature
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ Feature: Import CodeRunner questions
I need to import them

Background:
Given the following "users" exist:
Given the CodeRunner jobe sandbox is enabled
And the following "users" exist:
| username |
| teacher |
And the following "courses" exist:
Expand Down
Loading

0 comments on commit feb3b5d

Please sign in to comment.