Support DSL values in oracles

tested/internationalization/nl.yaml

@@ -77,7 +77,7 @@ nl:
         Contacteer de lesgever.
       unsupported: "Evaluation in %{lang} wordt niet ondersteund."
       unknown:
-        compilation: "Ongekende compilatie fout"
+        compilation: "Ongekende compilatiefout"
       produced:
         stderr: "De evaluatiecode produceerde volgende uitvoer op stderr:"
         stdout: "De evaluatiecode produceerde volgende uitvoer op stdout:"
@@ -86,7 +86,7 @@ nl:
   languages:
     config:
       unknown:
-        compilation: "Ongekende compilatie fout"
+        compilation: "Ongekende compilatiefout"
     generator:
       missing:
         input: "Geen invoer gevonden."

tested/judge/programmed.py

@@ -22,7 +22,8 @@
     generate_custom_evaluator,
     generate_statement,
 )
-from tested.serialisation import BooleanEvalResult, EvalResult, Value
+from tested.oracles.common import BooleanEvalResult
+from tested.serialisation import Value
 from tested.testsuite import CustomCheckOracle
 from tested.utils import get_identifier
 
@@ -34,7 +35,7 @@
     evaluator: CustomCheckOracle,
     expected: Value,
     actual: Value,
-) -> BaseExecutionResult | EvalResult:
+) -> BaseExecutionResult | BooleanEvalResult:
     """
     Run the custom evaluation. Concerning structure and execution, the custom
     oracle is very similar to the execution of the whole evaluation. It a
@@ -181,7 +182,7 @@
     oracle: CustomCheckOracle,
     expected: Value,
     actual: Value,
-) -> EvalResult:
+) -> BooleanEvalResult:
     """
     Run an evaluation in Python. While the templates are still used to generate
     code, they are not executed in a separate process, but inside python itself.
@@ -267,13 +268,12 @@
                 permission=Permission.STAFF,
             )
         )
-        return EvalResult(
+        return BooleanEvalResult(
             result=Status.INTERNAL_ERROR,
             readable_expected=None,
             readable_actual=None,
             messages=messages,
         )
 
-    assert isinstance(result_, BooleanEvalResult)
     result_.messages.extend(messages)
-    return result_.as_eval_result()
+    return result_

tested/languages/bash/generators.py

@@ -238,7 +238,7 @@ def convert_encoder(values: list[Value]) -> str:
 }}
 
 function send_value {{
-    echo "{{\\"type\\": \\"text\\", \\"data\\": $(json_escape "$1")}}"
+    echo "{{\\"type\\": \\"text\\", \\"data\\": $(json_escape "$1")}}␞"
 }}
 """
     for value in values:

tested/languages/c/generators.py

@@ -322,6 +322,6 @@ def convert_encoder(values: list[Value]) -> str:
 """
     for value in values:
         result += " " * 4 + f"write_value(stdout, {convert_value(value)});\n"
-        result += " " * 4 + 'printf("\\n");\n'
+        result += " " * 4 + 'printf("␞");\n'
     result += "}\n"
     return result

tested/languages/c/templates/evaluation_result.h

@@ -15,6 +15,8 @@ typedef struct EvaluationResult {
     char* readableActual;
     size_t nrOfMessages;
     Message** messages;
+    char* dslExpected;
+    char* dslActual;
 } EvaluationResult;
 
 

tested/languages/csharp/generators.py

@@ -478,7 +478,7 @@ def convert_encoder(values: list[Value]) -> str:
 
     for value in values:
         result += " " * 6 + f"Values.WriteValue(stdout, {convert_value(value)});"
-        result += " " * 6 + 'stdout.Write("\\n");\n'
+        result += " " * 6 + 'stdout.Write("␞");\n'
 
     result += "}\n}\n}\n"
     return result

tested/languages/csharp/templates/EvaluationResult.cs

@@ -3,7 +3,7 @@
 
 namespace Tested {
   public record Message(string Description, string Format = "text", string? Permission = null);
-  public record EvaluationResult(bool Result, string? ReadableExpected = null, string? ReadableActual = null, List<Message>? Messages = null)
+  public record EvaluationResult(bool Result, string? ReadableExpected = null, string? ReadableActual = null, List<Message>? Messages = null, string? DslExpected = null, string? DslActual = null)
   {
     public List<Message> Messages { get; init; } = Messages ?? new List<Message>();
   }

tested/languages/haskell/generators.py

@@ -350,5 +350,5 @@ def convert_encoder(values: list[Value]) -> str:
 
     for value in values:
         result += indent + f"sendValueH stdout ({convert_value(value)})\n"
-        result += indent + 'putStr "\\n"\n'
+        result += indent + 'putStr "␞"\n'
     return result

tested/languages/haskell/templates/EvaluationUtils.hs

@@ -11,7 +11,9 @@ data EvaluationResult = EvaluationResult {
     result :: Bool,
     readableExpected :: Maybe (String),
     readableActual :: Maybe (String),
-    messages :: [Message]
+    messages :: [Message],
+    dslExpected :: Maybe (String),
+    dslActual :: Maybe (String)
 } deriving Show
 
 message description = Message {
@@ -24,5 +26,7 @@ evaluationResult = EvaluationResult {
     result = False,
     readableExpected = Nothing,
     readableActual = Nothing,
+    dslExpected = Nothing,
+    dslActual = Nothing,
     messages = []
 }

tested/languages/java/generators.py

@@ -450,7 +450,7 @@ def convert_encoder(values: list[Value]) -> str:
 
     for value in values:
         result += " " * 8 + f"Values.send(writer, {convert_value(value)});\n"
-        result += " " * 8 + 'writer.write("\\n");\n'
+        result += " " * 8 + 'writer.write("␞");\n'
 
     result += " " * 8 + "writer.close();\n"
     result += " " * 4 + "}\n"

tested/languages/java/templates/EvaluationResult.java

@@ -6,12 +6,16 @@ public class EvaluationResult {
     public final boolean result;
     public final String readableExpected;
     public final String readableActual;
+    public final String dslExpected;
+    public final String dslActual;
     public final List<Message> messages;
 
-    private EvaluationResult(boolean result, String readableExpected, String readableActual, List<Message> messages) {
+    private EvaluationResult(boolean result, String readableExpected, String readableActual, String dslExpected, String dslActual, List<Message> messages) {
         this.result = result;
         this.readableExpected = readableExpected;
         this.readableActual = readableActual;
+        this.dslExpected = dslExpected;
+        this.dslActual = dslActual;
         this.messages = messages;
     }
 
@@ -44,6 +48,8 @@ public static class Builder {
         private final boolean result;
         private String readableExpected = null;
         private String readableActual = null;
+        private String dslExpected = null;
+        private String dslActual = null;
         private final List<Message> messages = new ArrayList<>();
 
         private Builder(boolean result) {
@@ -59,6 +65,16 @@ public Builder withReadableActual(String readableActual) {
             this.readableActual = readableActual;
             return this;
         }
+
+        public Builder withDslExpected(String dslExpected) {
+            this.dslExpected = dslExpected;
+            return this;
+        }
+
+        public Builder withDslActual(String dslActual) {
+            this.dslActual = dslActual;
+            return this;
+        }
 
         public Builder withMessages(List<Message> messages) {
             this.messages.addAll(messages);
@@ -71,7 +87,7 @@ public Builder withMessage(Message message) {
         }
 
         public EvaluationResult build() {
-            return new EvaluationResult(result, readableExpected, readableActual, messages);
+            return new EvaluationResult(result, readableExpected, readableActual, dslExpected, dslActual, messages);
         }
     }
 }

tested/languages/java/templates/Values.java

@@ -128,9 +128,13 @@ private static List<String> internalEncode(Object value) {
 
     private static String encode(Object value) {
         var typeAndData = internalEncode(value);
-        return "{ \"data\": " + typeAndData.get(1) + "," +
-                " \"type\": \"" + typeAndData.get(0) + "\", " +
-                " \"diagnostic\": " + typeAndData.get(2) + "}";
+        return """
+               {
+                 "data": %s,
+                 "type": "%s",
+                 "diagnostic": %s
+               }
+               """.formatted(typeAndData.get(1), typeAndData.get(0), typeAndData.get(2));
     }
 
     public static void send(PrintWriter writer, Object value) {
@@ -145,37 +149,57 @@ public static void sendException(PrintWriter writer, Throwable exception) {
         exception.printStackTrace(new PrintWriter(sw));
         var trace = sw.toString();
         var msg = exception.getMessage();
-        var result = "{ \"message\": \"" + escape(msg == null ? "" : msg) +
-                     "\", \"stacktrace\": \"" + escape(trace) +
-                     "\", \"type\": \"" + exception.getClass().getSimpleName() + "\"}";
+        String result = """
+                {
+                  "message": %s,
+                  "stacktrace": %s,
+                  "type": "%s"
+                }
+                """.formatted(asJson(msg), asJson(trace), exception.getClass().getSimpleName());
         writer.write(result);
     }
 
     private static String convertMessage(EvaluationResult.Message message) {
-        String result = "{" +
-            "\"description\": \"" + message.description + "\"," +
-            "\"format\": \"" + message.format + "\"";
-        if (message.permission instanceof String) {
-            result += ", \"permission\": \"" + message.permission + "\"";
-        }
-        return result + "}";
+        var description = asJson(message.description);
+        var format = asJson(message.format);
+        var permission = asJson(message.permission);
+
+        return """
+                {
+                  "description": %s,
+                  "format": %s,
+                  "permission": %s
+                }
+                """.formatted(description, format, permission);
     }
-
-    public static void evaluated(PrintWriter writer,
-                                 boolean result, String expected, String actual, Collection<EvaluationResult.Message> messages) {
-        List<String> converted = messages.stream().map(Values::convertMessage).collect(Collectors.toList());
-        String builder = "{" +
-            "\"result\": " +
-            result +
-            ", \"readable_expected\": \"" + expected + "\"" +
-            ", \"readable_actual\": \"" + actual + "\"" +
-            ", \"messages\": [" +
-            String.join(", ", converted) +
-            "]}";
-        writer.print(builder);
+
+    private static String asJson(String value) {
+        if (value == null) {
+            return "null";
+        } else {
+            return "\"%s\"".formatted(escape(value));
+        }
     }
 
     public static void sendEvaluated(PrintWriter writer, EvaluationResult r) {
-        evaluated(writer, r.result, r.readableExpected, r.readableActual, r.messages);
+        List<String> converted = r.messages.stream().map(Values::convertMessage).collect(Collectors.toList());
+        var readableExpected = asJson(r.readableExpected);
+        var readableActual = asJson(r.readableActual);
+        var dslExpected = asJson(r.dslExpected);
+        var dslActual = asJson(r.dslActual);
+        var messages = String.join(", ", converted);
+
+        String result = """
+                {
+                  "result": %b,
+                  "readable_expected": %s,
+                  "readable_actual": %s,
+                  "dsl_expected": %s,
+                  "dsl_actual": %s,
+                  "messages": [%s]
+                }
+                """.formatted(r.result, readableExpected, readableActual, dslExpected, dslActual, messages);
+
+        writer.print(result);
     }
 }

tested/languages/javascript/generators.py

@@ -275,6 +275,6 @@ def convert_encoder(values: list[Value]) -> str:
 
     for value in values:
         result += f"values.sendValue(process.stdout.fd, {convert_value(value)});\n"
-        result += 'fs.writeSync(process.stdout.fd, "\\n");\n'
+        result += 'fs.writeSync(process.stdout.fd, "␞");\n'
 
     return result

tested/languages/kotlin/generators.py

@@ -422,7 +422,7 @@ def convert_encoder(values: list[Value]) -> str:
 
     for value in values:
         result += " " * 4 + f"valuesSend(writer, {convert_value(value)})\n"
-        result += " " * 4 + 'writer.write("\\n")\n'
+        result += " " * 4 + 'writer.write("␞")\n'
 
     result += " " * 4 + "writer.close()\n"
     result += "}\n"

tested/languages/kotlin/templates/EvaluationResult.kt

@@ -2,6 +2,8 @@ class EvaluationResult private constructor(
         val result: Boolean,
         val readableExpected: String?,
         val readableActual: String?,
+        val dslExpected: String?,
+        val dslActual: String?,
         val messages: List<Message>
 ) {
     class Message(val description: String,
@@ -11,7 +13,9 @@ class EvaluationResult private constructor(
     class Builder(
             private val result: Boolean,
             private var readableActual: String? = null,
-            private var readableExpected: String? = null
+            private var readableExpected: String? = null,
+            private var dslActual: String? = null,
+            private var dslExpected: String? = null
     ) {
         private val messages: MutableList<Message> = ArrayList()
 
@@ -25,6 +29,16 @@ class EvaluationResult private constructor(
             return this
         }
 
+        fun withDslActual(dslActual: String?): Builder {
+            this.dslActual = dslActual
+            return this
+        }
+
+        fun withDslExpected(dslExpected: String?): Builder {
+            this.dslExpected = dslExpected
+            return this
+        }
+
         fun withMessages(messages: List<Message>): Builder {
             this.messages += messages
             return this
@@ -37,7 +51,7 @@ class EvaluationResult private constructor(
 
         fun build(): EvaluationResult {
             return EvaluationResult(result, readableExpected,
-                    readableActual, messages)
+                    readableActual, dslExpected, dslActual, messages)
         }
     }
 }