Integrated code lifecycle: Add Java blackbox exercise template (#10118)

ls1intum · Feb 5, 2025 · 5babeab · 5babeab
1 parent 445920e
commit 5babeab
Show file tree

Hide file tree

Showing 12 changed files with 443 additions and 160 deletions.
diff --git a/src/main/java/de/tum/cit/aet/artemis/buildagent/dto/CustomFeedback.java b/src/main/java/de/tum/cit/aet/artemis/buildagent/dto/CustomFeedback.java
@@ -0,0 +1,10 @@
+package de.tum.cit.aet.artemis.buildagent.dto;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public record CustomFeedback(@JsonProperty("name") String name, @JsonProperty("successful") boolean successful, @JsonProperty("message") String message) {
+
+    public String getMessage() {
+        return message != null ? message : "";
+    }
+}
diff --git a/src/main/java/de/tum/cit/aet/artemis/buildagent/service/BuildJobExecutionService.java b/src/main/java/de/tum/cit/aet/artemis/buildagent/service/BuildJobExecutionService.java
@@ -1,6 +1,5 @@
 package de.tum.cit.aet.artemis.buildagent.service;
 
-import static de.tum.cit.aet.artemis.buildagent.service.TestResultXmlParser.processTestResultFile;
 import static de.tum.cit.aet.artemis.core.config.Constants.CHECKED_OUT_REPOS_TEMP_DIR;
 import static de.tum.cit.aet.artemis.core.config.Constants.LOCALCI_RESULTS_DIRECTORY;
 import static de.tum.cit.aet.artemis.core.config.Constants.LOCALCI_WORKING_DIRECTORY;
@@ -44,6 +43,8 @@
 import de.tum.cit.aet.artemis.buildagent.dto.BuildResult;
 import de.tum.cit.aet.artemis.buildagent.dto.LocalCIJobDTO;
 import de.tum.cit.aet.artemis.buildagent.dto.LocalCITestJobDTO;
+import de.tum.cit.aet.artemis.buildagent.service.parser.CustomFeedbackParser;
+import de.tum.cit.aet.artemis.buildagent.service.parser.TestResultXmlParser;
 import de.tum.cit.aet.artemis.core.exception.EntityNotFoundException;
 import de.tum.cit.aet.artemis.core.exception.GitException;
 import de.tum.cit.aet.artemis.core.exception.LocalCIException;
@@ -406,20 +407,25 @@ private BuildResult parseTestResults(TarArchiveInputStream testResultsTarInputSt
             }
 
             // Read the contents of the tar entry as a string.
-            String fileString = readTarEntryContent(testResultsTarInputStream);
+            String fileContent = readTarEntryContent(testResultsTarInputStream);
             // Get the file name of the tar entry.
             String fileName = getFileName(tarEntry);
 
             try {
                 // Check if the file is a static code analysis report file
                 if (StaticCodeAnalysisTool.getToolByFilePattern(fileName).isPresent()) {
-                    processStaticCodeAnalysisReportFile(fileName, fileString, staticCodeAnalysisReports, buildJobId);
+                    processStaticCodeAnalysisReportFile(fileName, fileContent, staticCodeAnalysisReports, buildJobId);
                 }
                 else {
                     // ugly workaround because in swift result files \n\t breaks the parsing
-                    var testResultFileString = fileString.replace("\n\t", "");
+                    var testResultFileString = fileContent.replace("\n\t", "");
                     if (!testResultFileString.isBlank()) {
-                        processTestResultFile(testResultFileString, failedTests, successfulTests);
+                        if (fileName.endsWith(".xml")) {
+                            TestResultXmlParser.processTestResultFile(testResultFileString, failedTests, successfulTests);
+                        }
+                        else if (fileName.endsWith(".json")) {
+                            CustomFeedbackParser.processTestResultFile(fileName, testResultFileString, failedTests, successfulTests);
+                        }
                     }
                     else {
                         String msg = "The file " + fileName + " does not contain any testcases.";
@@ -446,7 +452,7 @@ private boolean isValidTestResultFile(TarArchiveEntry tarArchiveEntry) {
         String result = (lastIndexOfSlash != -1 && lastIndexOfSlash + 1 < name.length()) ? name.substring(lastIndexOfSlash + 1) : name;
 
         // Java test result files are named "TEST-*.xml", Python test result files are named "*results.xml".
-        return !tarArchiveEntry.isDirectory() && (result.endsWith(".xml") && !result.equals("pom.xml") || result.endsWith(".sarif"));
+        return !tarArchiveEntry.isDirectory() && (result.endsWith(".xml") && !result.equals("pom.xml") || result.endsWith(".json") || result.endsWith(".sarif"));
     }
 
     /**

diff --git a/src/main/java/de/tum/cit/aet/artemis/buildagent/service/parser/CustomFeedbackParser.java b/src/main/java/de/tum/cit/aet/artemis/buildagent/service/parser/CustomFeedbackParser.java
@@ -0,0 +1,66 @@
+package de.tum.cit.aet.artemis.buildagent.service.parser;
+
+import java.io.IOException;
+import java.util.InvalidPropertiesFormatException;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import de.tum.cit.aet.artemis.buildagent.dto.CustomFeedback;
+import de.tum.cit.aet.artemis.buildagent.dto.LocalCITestJobDTO;
+
+public final class CustomFeedbackParser {
+
+    private static final Logger log = LoggerFactory.getLogger(CustomFeedbackParser.class);
+
+    private static final ObjectMapper mapper = new ObjectMapper();
+
+    private CustomFeedbackParser() {
+
+    }
+
+    /**
+     * Parses the test result file and extracts failed and successful tests.
+     *
+     * @param fileName             The name of the result file. Needs to be present.
+     * @param testResultFileString The content of the test result file as a String.
+     * @param failedTests          A list of failed tests. This list will be populated by the method.
+     * @param successfulTests      A list of successful tests. This list will be populated by the method.
+     */
+    public static void processTestResultFile(final String fileName, final String testResultFileString, final List<LocalCITestJobDTO> failedTests,
+            final List<LocalCITestJobDTO> successfulTests) {
+        final CustomFeedback feedback;
+        try {
+            feedback = mapper.readValue(testResultFileString, CustomFeedback.class);
+            validateCustomFeedback(fileName, feedback);
+        }
+        catch (IOException e) {
+            log.error("Error during custom Feedback creation. {}", e.getMessage(), e);
+            return;
+        }
+        List<LocalCITestJobDTO> toAddFeedbackTo = feedback.successful() ? successfulTests : failedTests;
+        toAddFeedbackTo.add(new LocalCITestJobDTO(feedback.name(), List.of(feedback.getMessage())));
+    }
+
+    /**
+     * Checks that the custom feedback has a valid format
+     * <p>
+     * A custom feedback has to have a non-empty, non only-whitespace name to be able to identify it in Artemis.
+     * If it is not successful, there has to be a message explaining a reason why this is the case.
+     *
+     * @param fileName where the custom feedback was read from.
+     * @param feedback the custom feedback to validate.
+     * @throws InvalidPropertiesFormatException if one of the invariants described above does not hold.
+     */
+    private static void validateCustomFeedback(final String fileName, final CustomFeedback feedback) throws InvalidPropertiesFormatException {
+        if (feedback.name() == null || feedback.name().trim().isEmpty()) {
+            throw new InvalidPropertiesFormatException(String.format("Custom feedback from file %s needs to have a name attribute.", fileName));
+        }
+        if (!feedback.successful() && (feedback.message() == null || feedback.message().trim().isEmpty())) {
+            throw new InvalidPropertiesFormatException(String.format("Custom non-success feedback from file %s needs to have a message", fileName));
+        }
+    }
+}
diff --git a/...ildagent/service/TestResultXmlParser.java → ...t/service/parser/TestResultXmlParser.java b/...ildagent/service/TestResultXmlParser.java → ...t/service/parser/TestResultXmlParser.java
@@ -1,4 +1,4 @@
-package de.tum.cit.aet.artemis.buildagent.service;
+package de.tum.cit.aet.artemis.buildagent.service.parser;
 
 import java.io.IOException;
 import java.util.Collections;

diff --git a/...cit/aet/artemis/programming/service/localci/LocalCIProgrammingLanguageFeatureService.java b/...cit/aet/artemis/programming/service/localci/LocalCIProgrammingLanguageFeatureService.java
@@ -24,6 +24,7 @@
 import static de.tum.cit.aet.artemis.programming.domain.ProjectType.FACT;
 import static de.tum.cit.aet.artemis.programming.domain.ProjectType.GCC;
 import static de.tum.cit.aet.artemis.programming.domain.ProjectType.GRADLE_GRADLE;
+import static de.tum.cit.aet.artemis.programming.domain.ProjectType.MAVEN_BLACKBOX;
 import static de.tum.cit.aet.artemis.programming.domain.ProjectType.MAVEN_MAVEN;
 import static de.tum.cit.aet.artemis.programming.domain.ProjectType.PLAIN;
 import static de.tum.cit.aet.artemis.programming.domain.ProjectType.PLAIN_GRADLE;
@@ -65,7 +66,7 @@ protected Map<ProgrammingLanguage, ProgrammingLanguageFeature> getSupportedProgr
         programmingLanguageFeatures.put(GO, new ProgrammingLanguageFeature(GO, false, false, true, true, false, List.of(), true));
         programmingLanguageFeatures.put(HASKELL, new ProgrammingLanguageFeature(HASKELL, true, false, false, false, true, List.of(), true));
         programmingLanguageFeatures.put(JAVA,
-                new ProgrammingLanguageFeature(JAVA, true, true, true, true, false, List.of(PLAIN_GRADLE, GRADLE_GRADLE, PLAIN_MAVEN, MAVEN_MAVEN), true));
+                new ProgrammingLanguageFeature(JAVA, true, true, true, true, false, List.of(PLAIN_GRADLE, GRADLE_GRADLE, PLAIN_MAVEN, MAVEN_MAVEN, MAVEN_BLACKBOX), true));
         programmingLanguageFeatures.put(JAVASCRIPT, new ProgrammingLanguageFeature(JAVASCRIPT, false, false, true, false, false, List.of(), true));
         programmingLanguageFeatures.put(KOTLIN, new ProgrammingLanguageFeature(KOTLIN, false, false, true, true, false, List.of(), true));
         programmingLanguageFeatures.put(MATLAB, new ProgrammingLanguageFeature(MATLAB, false, false, false, false, false, List.of(), true));

diff --git a/src/main/resources/templates/aeolus/java/plain_maven_blackbox.sh b/src/main/resources/templates/aeolus/java/plain_maven_blackbox.sh
@@ -6,36 +6,48 @@ build () {
   mvn -B clean compile
 }
 
-checkers () {
-  echo '⚙️ executing checkers'
-  # all java files in the assignment folder should have maximal line length 80
-  pipeline-helper line-length -l 80 -s ${studentParentWorkingDirectoryName}/ -e java
-  # checks that the file exists and is not empty for non gui programs
-  pipeline-helper file-exists ${studentParentWorkingDirectoryName}/Tests.txt
-
+main_method_checker () {
+  echo '⚙️ executing main_method_checker'
   main_checker_output=$(pipeline-helper main-method -s target/classes)
 
-  IFS=$'\n' read -rd '' -a main_checker_lines <<< "${main_checker_output}"
+  line_count="$(echo "$main_checker_output" | wc -l)"
+  main_class="$(echo "$main_checker_output" | head -n1)"
 
-  if [ ${#main_checker_lines[@]} -eq 2 ]; then
-    export MAIN_CLASS=${main_checker_lines[0]}
+  if [ "${line_count}" -eq 2 ]; then
+    echo main method found in file: "${main_class}"
+    sed -i "s#MAIN_CLASS#${main_class}#" testsuite/config/default.exp
   else
+    echo "no main method found. quitting the test run."
     exit 1
   fi
+}
 
-  JAVA_FLAGS="-Djdk.console=java.base"
+custom_checkers () {
+  echo '⚙️ executing custom_checkers'
+  # all java files in the assignment folder should have maximal line length 80
+  pipeline-helper line-length -l 80 -s ${studentParentWorkingDirectoryName}/ -e java
+  # checks that the file exists and is not empty for non gui programs
+  pipeline-helper file-exists ${studentParentWorkingDirectoryName}/Tests.txt
+}
+
+replace_script_variables () {
+  echo '⚙️ executing replace_script_variables'
+  local JAVA_FLAGS="-Djdk.console=java.base"
+  local testfiles_base_path="./testsuite/testfiles"
+  local tool=$(find testsuite -name "*.tests" -type d -printf "%f" | sed 's#.tests$##')
 
   sed -i "s#JAVA_FLAGS#${JAVA_FLAGS}#;s#CLASSPATH#../target/classes#" testsuite/config/default.exp
-  sed -i "s#MAIN_CLASS#${MAIN_CLASS}#" testsuite/config/default.exp
-  export testfiles_base_path="./testsuite/testfiles"
-  export tool=$(find testsuite -name "*.tests" -type d -printf "%f" | sed 's#.tests$##')
   sed -i "s#TESTFILES_DIRECTORY#../${testfiles_base_path}#" testsuite/${tool}.tests/*.exp
 }
 
-secrettests () {
-  echo '⚙️ executing secrettests'
-  export tool=$(find testsuite -name "*.tests" -type d -printf "%f" | sed 's#.tests$##')
-  export step="secret"
+secret_tests () {
+  echo '⚙️ executing secret_tests'
+  if [ ! -d ./testsuite ]; then
+    echo "skipping secret tests because the testsuite folder does not exist."
+  fi
+
+  local step="secret"
+  local tool=$(find testsuite -name "*.tests" -type d -printf "%f" | sed 's#.tests$##')
   cd testsuite || exit
   rm ${tool}.log || true
   timeout 60s runtest --tool ${tool} ${step}.exp || true
@@ -46,34 +58,34 @@ secrettests () {
     rm "${secretExp}"
   fi
 
-  export testfiles_base_path="./testsuite/testfiles"
-
-  if [ -f "${testfiles_base_path}/secret" ]; then
-    rm "${testfiles_base_path}/secret"
+  if [ -d "${testfiles_base_path}/secret" ]; then
+    rm -rf "${testfiles_base_path}/secret"
   fi
 }
 
-publictests () {
-  echo '⚙️ executing publictests'
-  export tool=$(find testsuite -name "*.tests" -type d -printf "%f" | sed 's#.tests$##')
-  sed -i "s#TESTFILES_DIRECTORY#../${testfiles_base_path}#" testsuite/${tool}.tests/*.exp
+public_tests () {
+  echo '⚙️ executing public_tests'
+  if [ ! -d ./testsuite ]; then
+    echo "skipping public tests because the testsuite folder does not exist."
+  fi
 
-  export tool=$(find testsuite -name "*.tests" -type d -printf "%f" | sed 's#.tests$##')
-  export step="public"
+  local step="public"
+  local tool=$(find testsuite -name "*.tests" -type d -printf "%f" | sed 's#.tests$##')
   cd testsuite || exit
   rm ${tool}.log || true
   timeout 60s runtest --tool ${tool} ${step}.exp || true
   cd ..
   pipeline-helper -o customFeedbacks dejagnu -n "dejagnu[${step}]" -l testsuite/${tool}.log
 }
 
-advancedtests () {
-  echo '⚙️ executing advancedtests'
-  export testfiles_base_path="./testsuite/testfiles"
-  export tool=$(find testsuite -name "*.tests" -type d -printf "%f" | sed 's#.tests$##')
-  sed -i "s#TESTFILES_DIRECTORY#../${testfiles_base_path}#" testsuite/${tool}.tests/*.exp
+advanced_tests () {
+  echo '⚙️ executing advanced_tests'
+  if [ ! -d ./testsuite ]; then
+    echo "skipping advanced tests because the testsuite folder does not exist."
+  fi
 
-  export step="advanced"
+  local step="advanced"
+  local tool=$(find testsuite -name "*.tests" -type d -printf "%f" | sed 's#.tests$##')
   cd testsuite || exit
   rm ${tool}.log || true
   timeout 60s runtest --tool ${tool} ${step}.exp || true
@@ -90,13 +102,17 @@ main () {
   cd "${AEOLUS_INITIAL_DIRECTORY}"
   bash -c "source ${_script_name} aeolus_sourcing; build"
   cd "${AEOLUS_INITIAL_DIRECTORY}"
-  bash -c "source ${_script_name} aeolus_sourcing; checkers"
+  bash -c "source ${_script_name} aeolus_sourcing; main_method_checker"
+  cd "${AEOLUS_INITIAL_DIRECTORY}"
+  bash -c "source ${_script_name} aeolus_sourcing; custom_checkers"
+  cd "${AEOLUS_INITIAL_DIRECTORY}"
+  bash -c "source ${_script_name} aeolus_sourcing; replace_script_variables"
   cd "${AEOLUS_INITIAL_DIRECTORY}"
-  bash -c "source ${_script_name} aeolus_sourcing; secrettests"
+  bash -c "source ${_script_name} aeolus_sourcing; secret_tests"
   cd "${AEOLUS_INITIAL_DIRECTORY}"
-  bash -c "source ${_script_name} aeolus_sourcing; publictests"
+  bash -c "source ${_script_name} aeolus_sourcing; public_tests"
   cd "${AEOLUS_INITIAL_DIRECTORY}"
-  bash -c "source ${_script_name} aeolus_sourcing; advancedtests"
+  bash -c "source ${_script_name} aeolus_sourcing; advanced_tests"
 }
 
 main "${@}"