From 81fb9df98e90e710c87c9e5c44458577fc69f2a8 Mon Sep 17 00:00:00 2001 From: Andrey Vihrov Date: Fri, 26 Mar 2021 21:33:06 +0200 Subject: [PATCH] Add functional tests for user test evaluation with graders This also adds checks that user test evaluation in fact succeeds. --- cmstestsuite/Test.py | 73 ++++++++++++++----- cmstestsuite/Tests.py | 9 ++- .../tasks/batch_fileio_managed/__init__.py | 2 + .../tasks/batch_fileio_managed/code/grader.c | 2 + .../tasks/batch_fileio_managed/code/task.h | 0 5 files changed, 64 insertions(+), 22 deletions(-) create mode 100644 cmstestsuite/tasks/batch_fileio_managed/code/task.h diff --git a/cmstestsuite/Test.py b/cmstestsuite/Test.py index a4919215b7..07e912fcd1 100644 --- a/cmstestsuite/Test.py +++ b/cmstestsuite/Test.py @@ -3,7 +3,7 @@ # Contest Management System - http://cms-dev.github.io/ # Copyright © 2012 Bernard Blackham # Copyright © 2014-2017 Stefano Maggiolo -# Copyright © 2020 Andrey Vihrov +# Copyright © 2020-2021 Andrey Vihrov # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -20,7 +20,7 @@ import os import re -from abc import ABCMeta, abstractmethod +from abc import ABC, abstractmethod from cms.grading.languagemanager import get_language from cmstestsuite.functionaltestframework import FunctionalTestFramework @@ -30,7 +30,7 @@ class TestFailure(Exception): pass -class Check(metaclass=ABCMeta): +class Check(ABC): @abstractmethod def check(self, *args, **kwargs): pass @@ -133,9 +133,22 @@ def __init__(self): "Execution failed because the return code was nonzero") +class CheckUserTest(ABC): + @abstractmethod + def check(self, *args, **kwargs): + pass + + +class CheckUserTestEvaluated(CheckUserTest): + def check(self, result_info): + if "Evaluated" not in result_info["status"]: + raise TestFailure("Expected a successful evaluation, got: %s" % + result_info["status"]) + + class Test: - def __init__(self, name, task, filenames, languages, checks, - user_tests=False, alt_filenames={}): + def __init__(self, name, *, task, filenames, alt_filenames={}, languages, + checks, user_tests=False, user_managers=[], user_checks=[]): self.framework = FunctionalTestFramework() self.name = name @@ -149,27 +162,44 @@ def __init__(self, name, task, filenames, languages, checks, self.submission_format = submission_format self.user_tests = user_tests + self.user_checks = user_checks + # Some tasks may require additional user test managers. + self.user_managers = user_managers + user_manager_format = list(e.strip() + for e in task.task_info.get("user_manager_format", "").split(",")) + self.user_manager_format = user_manager_format self.submission_id = {} self.user_test_id = {} - def _sources_names(self, language): - # Source files are stored under cmstestsuite/code/. - path = os.path.join(os.path.dirname(__file__), 'code') - - # Choose the correct file to submit. + def _filenames_for_language(self, language, filenames, alt_filenames={}): if language is not None: # First check if language is in alt_filenames. This allows to # submit different sources for languages that would otherwise # have matching source extensions. - filenames = self.alt_filenames.get(language, self.filenames) + filenames = alt_filenames.get(language, filenames) ext = get_language(language).source_extension - filenames = [filename.replace(".%l", ext) - for filename in filenames] + return [filename.replace(".%l", ext) for filename in filenames] else: - filenames = self.filenames + return filenames + + def _sources_names(self, language): + # Source files are stored under cmstestsuite/code/. + path = os.path.join(os.path.dirname(__file__), 'code') + + filenames = self._filenames_for_language(language, self.filenames, + self.alt_filenames) + full_paths = [os.path.join(path, filename) for filename in filenames] + + return full_paths + def _user_managers_names(self, language): + # Currently we select the same managers that are used for submission + # evaluation. These are located under /code/. + path = os.path.join(os.path.dirname(self.task_module.__file__), "code") + + filenames = self._filenames_for_language(language, self.user_managers) full_paths = [os.path.join(path, filename) for filename in filenames] return full_paths @@ -199,10 +229,11 @@ def wait(self, contest_id, language): raise def submit_user_test(self, task_id, user_id, language): - full_paths = self._sources_names(language) + submission_format = self.submission_format + self.user_manager_format + full_paths = self._sources_names(language) + \ + self._user_managers_names(language) self.user_test_id[language] = self.framework.cws_submit_user_test( - task_id, user_id, - self.submission_format, full_paths, language) + task_id, user_id, submission_format, full_paths, language) def wait_user_test(self, contest_id, language): # This means we were not able to submit, hence the error @@ -210,6 +241,10 @@ def wait_user_test(self, contest_id, language): if self.user_test_id.get(language) is None: return - # Wait for evaluation to complete. We do not do any other check. - self.framework.get_user_test_result( + # Wait for evaluation to complete. + result_info = self.framework.get_user_test_result( contest_id, self.user_test_id[language]) + + # Run checks. + for check in self.user_checks: + check.check(result_info) diff --git a/cmstestsuite/Tests.py b/cmstestsuite/Tests.py index 2544b37c94..7cf8416ac9 100644 --- a/cmstestsuite/Tests.py +++ b/cmstestsuite/Tests.py @@ -37,7 +37,7 @@ import cmstestsuite.tasks.twosteps as twosteps import cmstestsuite.tasks.twosteps_comparator as twosteps_comparator from cmstestsuite.Test import Test, CheckOverallScore, CheckCompilationFail, \ - CheckTimeout, CheckTimeoutWall, CheckNonzeroReturn + CheckTimeout, CheckTimeoutWall, CheckNonzeroReturn, CheckUserTestEvaluated LANG_CPP = "C++11 / g++" @@ -81,7 +81,8 @@ task=batch_fileio, filenames=['correct-freopen.%l'], languages=(LANG_C,), checks=[CheckOverallScore(100, 100)], - user_tests=True), + user_tests=True, + user_checks=[CheckUserTestEvaluated()]), Test('correct-stdio-inner-class', task=batch_stdio, filenames=['correct-stdio-inner-class.%l'], @@ -250,7 +251,9 @@ task=batch_fileio_managed, filenames=['managed-correct.%l'], languages=(LANG_C, LANG_CPP, LANG_PASCAL, LANG_PYTHON3, LANG_JAVA, LANG_C_SHARP), - checks=[CheckOverallScore(100, 100)]), + checks=[CheckOverallScore(100, 100)], + user_tests=True, user_managers=['grader.%l'], + user_checks=[CheckUserTestEvaluated()]), Test('managed-incorrect', task=batch_fileio_managed, filenames=['managed-incorrect.%l'], diff --git a/cmstestsuite/tasks/batch_fileio_managed/__init__.py b/cmstestsuite/tasks/batch_fileio_managed/__init__.py index 44afd5469f..7671073c99 100644 --- a/cmstestsuite/tasks/batch_fileio_managed/__init__.py +++ b/cmstestsuite/tasks/batch_fileio_managed/__init__.py @@ -23,6 +23,7 @@ "official_language": "", "submission_format_choice": "other", "submission_format": "batchfileiomanaged.%l", + "user_manager_format": "grader.%l", "time_limit_{{dataset_id}}": "0.5", "memory_limit_{{dataset_id}}": "128", "task_type_{{dataset_id}}": "Batch", @@ -35,6 +36,7 @@ } managers = [ + "task.h", "grader.c", "grader.cpp", "grader.pas", diff --git a/cmstestsuite/tasks/batch_fileio_managed/code/grader.c b/cmstestsuite/tasks/batch_fileio_managed/code/grader.c index ba3fbe4638..51d210f4ca 100644 --- a/cmstestsuite/tasks/batch_fileio_managed/code/grader.c +++ b/cmstestsuite/tasks/batch_fileio_managed/code/grader.c @@ -1,5 +1,7 @@ #include +#include "task.h" + extern int userfunc(int x); int main() { diff --git a/cmstestsuite/tasks/batch_fileio_managed/code/task.h b/cmstestsuite/tasks/batch_fileio_managed/code/task.h new file mode 100644 index 0000000000..e69de29bb2