Acceptance tests for JSON output during execution (#3423)

Also enhance tests for JSON output with Rebot.

atest/resources/TestCheckerLibrary.py

@@ -1,12 +1,15 @@
+import json
 import os
 import re
 from pathlib import Path
 
+from jsonschema import Draft202012Validator
 from xmlschema import XMLSchema
 
 from robot import utils
 from robot.api import logger
 from robot.libraries.BuiltIn import BuiltIn
+from robot.libraries.Collections import Collections
 from robot.result import (
     Break, Continue, Error, ExecutionResult, ExecutionResultBuilder, For,
     ForIteration, Group, If, IfBranch, Keyword, Result, ResultVisitor, Return,
@@ -147,7 +150,9 @@ class TestCheckerLibrary:
     ROBOT_LIBRARY_SCOPE = 'GLOBAL'
 
     def __init__(self):
-        self.schema = XMLSchema('doc/schema/result.xsd')
+        self.xml_schema = XMLSchema('doc/schema/result.xsd')
+        with open('doc/schema/result.json', encoding='UTF-8') as f:
+            self.json_schema = Draft202012Validator(json.load(f))
 
     def process_output(self, path: 'None|Path', validate: 'bool|None' = None):
         set_suite_variable = BuiltIn().set_suite_variable
@@ -177,18 +182,22 @@ def _validate_output(self, path):
         version = self._get_schema_version(path)
         if not version:
             raise ValueError('Schema version not found from XML output.')
-        if version != self.schema.version:
+        if version != self.xml_schema.version:
             raise ValueError(f'Incompatible schema versions. '
-                             f'Schema has `version="{self.schema.version}"` but '
+                             f'Schema has `version="{self.xml_schema.version}"` but '
                              f'output file has `schemaversion="{version}"`.')
-        self.schema.validate(path)
+        self.xml_schema.validate(path)
 
     def _get_schema_version(self, path):
         with open(path, encoding='UTF-8') as file:
             for line in file:
                 if line.startswith('<robot'):
                     return re.search(r'schemaversion="(\d+)"', line).group(1)
 
+    def validate_json_output(self, path: Path):
+        with path.open(encoding='UTF') as file:
+            self.json_schema.validate(json.load(file))
+
     def get_test_case(self, name):
         suite = BuiltIn().get_variable_value('${SUITE}')
         return self._get_test_from_suite(suite, name)
@@ -376,16 +385,26 @@ def check_log_message(self, item, expected, level='INFO', html=False, pattern=Fa
             b.should_be_equal(item.level, 'INFO' if level == 'HTML' else level, 'Wrong log level')
         b.should_be_equal(str(item.html), str(html or level == 'HTML'), 'Wrong HTML status')
 
-    def outputs_should_be_equal(self, output1, output2):
+    def outputs_should_contain_same_data(self, output1, output2, ignore_timestamps=False):
+        dictionaries_should_be_equal = Collections().dictionaries_should_be_equal
+        if ignore_timestamps:
+            ignore_keys = ['start_time', 'end_time', 'elapsed_time', 'timestamp']
+        else:
+            ignore_keys = None
         result1 = ExecutionResult(output1)
         result2 = ExecutionResult(output2)
-        should_be_equal = BuiltIn().should_be_equal
-        should_be_equal(result1.suite.to_dict(),
-                        result2.suite.to_dict())
-        should_be_equal(result1.statistics.to_dict(),
-                        result2.statistics.to_dict())
-        should_be_equal(result1.errors.messages.to_dicts(),
-                        result2.errors.messages.to_dicts())
+        dictionaries_should_be_equal(result1.suite.to_dict(),
+                                     result2.suite.to_dict(),
+                                     ignore_keys=ignore_keys)
+        dictionaries_should_be_equal(result1.statistics.to_dict(),
+                                     result2.statistics.to_dict(),
+                                     ignore_keys=ignore_keys)
+        # Use `zip(..., strict=True)` when Python 3.10 is minimum version.
+        assert len(result1.errors) == len(result2.errors)
+        for msg1, msg2 in zip(result1.errors, result2.errors):
+            dictionaries_should_be_equal(msg1.to_dict(),
+                                         msg2.to_dict(),
+                                         ignore_keys=ignore_keys)
 
 
 class ProcessResults(ResultVisitor):

atest/robot/output/json_output.robot

@@ -0,0 +1,41 @@
+*** Settings ***
+Documentation     JSON output is tested in detailed level using unit tests.
+Resource          atest_resource.robot
+
+*** Variables ***
+${JSON}           %{TEMPDIR}/output.json
+${XML}            %{TEMPDIR}/output.xml
+
+*** Test Cases ***
+JSON output contains same suite information as XML output
+    Run Tests    ${EMPTY}    misc
+    Copy File    ${OUTFILE}    ${XML}
+    Run Tests Without Processing Output    -o ${JSON}    misc
+    Outputs Should Contain Same Data    ${JSON}    ${XML}    ignore_timestamps=True
+
+JSON output structure
+    [Documentation]    Full JSON schema validation would be good, but it's too slow with big output files.
+    ...                The following test validates a smaller suite.
+    ${data} =    Evaluate    json.load(open($JSON, encoding='UTF-8'))
+    Lists Should Be Equal    ${data}    ${{['generator', 'generated', 'rpa', 'suite', 'statistics', 'errors']}}
+    Should Match       ${data}[generator]                     Robot ?.* (* on *)
+    Should Match       ${data}[generated]                     20??-??-??T??:??:??.??????
+    Should Be Equal    ${data}[rpa]                           ${False}
+    Should Be Equal    ${data}[suite][name]                   Misc
+    Should Be Equal    ${data}[suite][suites][1][name]        Everything
+    Should Be Equal    ${data}[statistics][total][skip]       ${3}
+    Should Be Equal    ${data}[statistics][tags][4][label]    f1
+    Should Be Equal    ${data}[statistics][suites][-1][id]    s1-s17
+    Should Be Equal    ${data}[errors][0][level]              ERROR
+
+JSON output matches schema
+    Run Tests Without Processing Output    -o OUT.JSON    misc/everything.robot
+    Validate JSON Output    ${OUTDIR}/OUT.JSON
+
+Invalid JSON output file
+    ${path} =    Normalize Path    ${JSON}
+    Remove File    ${path}
+    Create Directory    ${path}
+    Run Tests Without Processing Output    -o ${path}    misc/pass_and_fail.robot
+    Stderr Should Match    [[] ERROR ] Opening output file '${path}' failed: *${USAGE TIP}\n
+    [Teardown]    Remove Directory    ${JSON}

atest/robot/rebot/json_output_and_input.robot

@@ -7,12 +7,12 @@ ${XML}            %{TEMPDIR}/rebot.xml
 ${JSON}           %{TEMPDIR}/rebot.json
 
 *** Test Cases ***
-JSON output
-    Outputs should be equal    ${JSON}    ${XML}
+JSON output contains same suite information as XML output
+    Outputs Should Contain Same Data    ${JSON}    ${XML}
 
 JSON output structure
     [Documentation]    JSON schema validation would be good, but it's too slow with big output files.
-    ...                Unit tests do schema validation with smaller data and that ought to be enough.
+    ...                The following test validates a smaller suite and unit tests do schema validation as well.
     ${data} =    Evaluate    json.load(open($JSON, encoding='UTF-8'))
     Lists Should Be Equal    ${data}    ${{['generator', 'generated', 'rpa', 'suite', 'statistics', 'errors']}}
     Should Match       ${data}[generator]                     Rebot ?.* (* on *)
@@ -25,17 +25,21 @@ JSON output structure
     Should Be Equal    ${data}[statistics][suites][-1][id]    s1-s17
     Should Be Equal    ${data}[errors][0][level]              ERROR
 
+JSON output schema validation
+    Run Rebot Without Processing Output    --suite Everything --output %{TEMPDIR}/everything.json    ${JSON}
+    Validate JSON Output    %{TEMPDIR}/everything.json
+
 JSON input
     Run Rebot    ${EMPTY}    ${JSON}
-    Outputs should be equal    ${JSON}    ${OUTFILE}
+    Outputs Should Contain Same Data    ${JSON}    ${OUTFILE}
 
 JSON input combined
     Run Rebot    ${EMPTY}    ${XML} ${XML}
     Copy Previous Outfile    # Expected result
     Run Rebot    ${EMPTY}    ${JSON} ${XML}
-    Outputs should be equal    ${OUTFILE}    ${OUTFILE COPY}
+    Outputs Should Contain Same Data    ${OUTFILE}    ${OUTFILE COPY}
     Run Rebot    ${EMPTY}    ${JSON} ${JSON}
-    Outputs should be equal    ${OUTFILE}    ${OUTFILE COPY}
+    Outputs Should Contain Same Data    ${OUTFILE}    ${OUTFILE COPY}
 
 Invalid JSON input
     Create File    ${JSON}    bad

atest/testdata/misc/everything.robot

@@ -16,18 +16,19 @@ User keyword and RETURN
     ${value} =    User Keyword    value
     Should Be Equal    ${value}    return value
 
-Test documentation and tags
+Test documentation, tags and timeout
     [Documentation]    Hello, world!
     [Tags]    hello    world
+    [Timeout]    1 min
     No Operation
 
 Test setup and teardown
     [Setup]    Log    Library keyword
     Log    Body
     [Teardown]    User Keyword
 
-Keyword documentation and tags
-    Keyword documentation and tags
+Keyword Keyword documentation, tags and timeout
+    Keyword documentation, tags and timeout
 
 Keyword setup and teardown
     Keyword setup and teardown
@@ -101,9 +102,10 @@ User keyword
     Should Be Equal    ${arg}    value
     RETURN    return ${arg}
 
-Keyword documentation and tags
+Keyword documentation, tags and timeout
     [Documentation]    Hello, world!
     [Tags]    hello    world
+    [Timeout]    1 day
     No Operation
 
 Keyword setup and teardown