JUnitReportReporter should capture the test case output at the test c…

…ase level (#2875)

CHANGES.txt

@@ -1,4 +1,5 @@
 Current
+Fixed: GITHUB-2875: JUnitReportReporter should capture the test case output at the test case level
 Fixed: GITHUB-2771: After upgrading to TestNG 7.5.0, setting ITestResult.status to FAILURE doesn't fail the test anymore (Julien Herr & Krishnan Mahadevan)
 Fixed: GITHUB-2796: Option for onAfterClass to run after @AfterClass
 Fixed: GITHUB-2857: XmlTest index is not set for test suites invoked with YAML

testng-core/src/main/java/org/testng/reporters/JUnitReportReporter.java

@@ -130,22 +130,38 @@ public void generateReport(
 
       xsb.push(XMLConstants.TESTSUITE, p1);
       for (TestTag testTag : testCases) {
-        if (putElement(xsb, XMLConstants.TESTCASE, testTag.properties, testTag.childTag != null)) {
-          Properties p = new Properties();
-          safeSetProperty(p, XMLConstants.ATTR_MESSAGE, testTag.message);
-          safeSetProperty(p, XMLConstants.ATTR_TYPE, testTag.type);
-
-          if (putElement(xsb, testTag.childTag, p, testTag.stackTrace != null)) {
-            xsb.addCDATA(testTag.stackTrace);
-            xsb.pop(testTag.childTag);
+        boolean testCaseHasChildElements = testTag.childTag != null || testTag.sysOut != null;
+        if (putElement(xsb, XMLConstants.TESTCASE, testTag.properties, testCaseHasChildElements)) {
+
+          if (testTag.childTag != null) {
+            Properties p = new Properties();
+            safeSetProperty(p, XMLConstants.ATTR_MESSAGE, testTag.message);
+            safeSetProperty(p, XMLConstants.ATTR_TYPE, testTag.type);
+
+            if (putElement(xsb, testTag.childTag, p, testTag.stackTrace != null)) {
+              xsb.addCDATA(testTag.stackTrace);
+              xsb.pop(testTag.childTag);
+            }
+          }
+
+          // Add reporter output for each test case as a child system-out element of testcase.
+          if (testTag.sysOut != null) {
+            putElement(xsb, XMLConstants.SYSTEM_OUT, new Properties(), true);
+            xsb.addCDATA(testTag.sysOut);
+            xsb.pop(XMLConstants.SYSTEM_OUT);
           }
           xsb.pop(XMLConstants.TESTCASE);
         }
-        if (putElement(xsb, XMLConstants.SYSTEM_OUT, new Properties(), testTag.sysOut != null)) {
-          xsb.addCDATA(testTag.sysOut);
-          xsb.pop(XMLConstants.SYSTEM_OUT);
-        }
       }
+
+      // Add the full reporter output once as a child system-out element of testsuite.
+      List<String> output = Reporter.getOutput();
+      if ((!output.isEmpty())) {
+        putElement(xsb, XMLConstants.SYSTEM_OUT, new Properties(), true);
+        xsb.addCDATA(String.join("\n", output));
+        xsb.pop(XMLConstants.SYSTEM_OUT);
+      }
+
       xsb.pop(XMLConstants.TESTSUITE);
 
       String outputDirectory = defaultOutputDirectory + File.separator + "junitreports";

testng-core/src/test/java/test/junitreports/JUnitReportsTest.java

@@ -13,6 +13,7 @@
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Stream;
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
@@ -22,12 +23,16 @@
 import javax.xml.xpath.XPathConstants;
 import javax.xml.xpath.XPathFactory;
 import org.testng.ITestNGListener;
+import org.testng.Reporter;
 import org.testng.TestNG;
 import org.testng.annotations.Test;
 import org.testng.collections.Maps;
+import org.testng.reporters.XMLConstants;
 import org.testng.xml.XmlSuite;
 import org.testng.xml.XmlTest;
 import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
 import org.xml.sax.SAXException;
 import org.xml.sax.SAXParseException;
 import org.xml.sax.helpers.DefaultHandler;
@@ -131,12 +136,165 @@ public void testEnsureTestnameDoesnotAcceptNullValues() throws IOException {
 
   @Test
   public void ensureTestReportContainsValidSysOutContent() throws Exception {
+    Class<?> testClass = TestClassSample.class;
     Path outputDir = TestHelper.createRandomDirectory();
-    TestNG tng = createTests(outputDir, "suite", TestClassSample.class);
+    TestNG tng = createTests(outputDir, "suite", testClass);
     tng.setUseDefaultListeners(true);
+    Reporter.clear();
     tng.run();
+    Document doc = getJunitReport(outputDir, testClass);
+    XPath xPath = XPathFactory.newInstance().newXPath();
+
+    // Define a result for each test case in the test class that can be used to compare the expected
+    // and actual results.
+    class TestCaseResult {
+      private final String name;
+      private final String failureMessage;
+      private final String systemOut;
+
+      // Construct an expected result.
+      public TestCaseResult(String name, String failureMessage, String... systemOut) {
+        this.name = name;
+        this.failureMessage = failureMessage;
+        this.systemOut = systemOut.length > 0 ? String.join("\n", systemOut) : null;
+      }
+
+      // Construct from an actual testcase xml report element.
+      public TestCaseResult(Element actualTestCaseElement) {
+        name = actualTestCaseElement.getAttribute(XMLConstants.ATTR_NAME);
+
+        NodeList failureList = actualTestCaseElement.getElementsByTagName(XMLConstants.FAILURE);
+        NodeList systemOutList =
+            actualTestCaseElement.getElementsByTagName(XMLConstants.SYSTEM_OUT);
+        assertThat(failureList.getLength()).isLessThanOrEqualTo(1);
+        assertThat(systemOutList.getLength()).isLessThanOrEqualTo(1);
+
+        failureMessage =
+            1 == failureList.getLength()
+                ? ((Element) failureList.item(0)).getAttribute(XMLConstants.ATTR_MESSAGE)
+                : null;
+        systemOut =
+            1 == systemOutList.getLength() ? systemOutList.item(0).getTextContent().trim() : null;
+      }
+
+      public boolean matches(TestCaseResult testCaseResult) {
+        return toString().equals(testCaseResult.toString());
+      }
+
+      @Override
+      public String toString() {
+        return String.format(
+            "TestCaseResult{name='%s', failureMessage='%s', systemOut='%s'}",
+            name, failureMessage, systemOut);
+      }
+    }
+
+    // Build a list of expected test case results and describe their possible failure or system-out
+    // values.
+    List<TestCaseResult> expectedTestCaseResults =
+        Arrays.asList(
+            // Verify the system-out for the reporter testcase.
+            new TestCaseResult(
+                TestClassSample.TEST_METHOD_WITH_REPORTER,
+                null,
+                TestClassSample.MESSAGE_1,
+                TestClassSample.MESSAGE_2),
+            // Verify the system-out for multiple test cases of the same name from the data provider
+            // testcase.
+            new TestCaseResult(
+                TestClassSample.TEST_METHOD_WITH_DATA_PROVIDER_REPORTER,
+                null,
+                TestClassSample.MESSAGE_3),
+            // Verify the system-out for multiple test cases of the same name from the data provider
+            // testcase.
+            new TestCaseResult(
+                TestClassSample.TEST_METHOD_WITH_DATA_PROVIDER_REPORTER,
+                null,
+                TestClassSample.MESSAGE_4),
+            // Verify that a test case can include system-out and failure elements.
+            new TestCaseResult(
+                TestClassSample.TEST_METHOD_FAIL_WITH_REPORTER,
+                TestClassSample.MESSAGE_FAIL,
+                TestClassSample.MESSAGE_5),
+            // Verify that a test case can have a failure element without system-out.
+            new TestCaseResult(
+                TestClassSample.TEST_METHOD_FAIL_NO_REPORTER, TestClassSample.MESSAGE_FAIL),
+            // Verify that a test case without any Reporter logs does not include a system-out
+            // element.
+            new TestCaseResult(TestClassSample.TEST_METHOD_NO_REPORTER, null));
+
+    // Verify that the count of actual xml testcase elements matches the count of expected test case
+    // results.
+    NodeList actualTestcases = doc.getElementsByTagName(XMLConstants.TESTCASE);
+    assertThat(actualTestcases.getLength()).isEqualTo(expectedTestCaseResults.size());
+    // Verify that each actual xml testcase element matches exactly one expected test case result.
+    for (int i = 0; i < actualTestcases.getLength(); i++) {
+      Element actualTestCaseElement = (Element) actualTestcases.item(i);
+      TestCaseResult actualTestCaseResult = new TestCaseResult(actualTestCaseElement);
+      long actualMatchCount =
+          expectedTestCaseResults.stream()
+              .filter(
+                  expectedTestCaseResult -> expectedTestCaseResult.matches(actualTestCaseResult))
+              .count();
+      assertThat(actualMatchCount)
+          .withFailMessage(
+              String.format(
+                  "Could not find an expected result for actual test case result %s",
+                  actualTestCaseResult))
+          .isEqualTo(1);
+    }
+
+    // Verify the actual full system-out for the testsuite which includes output lines from
+    // before, after, and test case methods.
+    String actualFullOutputExpression = "//testsuite/system-out";
+    String actualFullOutputData =
+        ((String) xPath.compile(actualFullOutputExpression).evaluate(doc, XPathConstants.STRING))
+            .trim();
+    List<String> actualFullOutputList = Arrays.asList(actualFullOutputData.split("\n"));
+
+    String[] expectedTestCaseOutputList =
+        expectedTestCaseResults.stream()
+            .flatMap(
+                result ->
+                    null == result.systemOut
+                        ? Stream.empty()
+                        : Arrays.stream(result.systemOut.split("\n")))
+            .toArray(String[]::new);
+    int expectedFullOutputCount = expectedTestCaseOutputList.length + 2;
+
+    // Verify that the count of actual xml testsuite system-out lines matches the count of expected
+    // output lines.
+    assertThat(actualFullOutputList.size()).isEqualTo(expectedFullOutputCount);
+
+    // Verify that before and after messages are at the beginning and end of the testsuite
+    // system-out.
+    assertThat(actualFullOutputList.get(0)).isEqualTo(TestClassSample.MESSAGE_BEFORE);
+    assertThat(actualFullOutputList.get(expectedFullOutputCount - 1))
+        .isEqualTo(TestClassSample.MESSAGE_AFTER);
+
+    // The order of messages from the test cases depends on the order the test cases were run so
+    // allow the check to be in any order.
+    assertThat(actualFullOutputList.subList(1, expectedFullOutputCount - 1))
+        .containsExactlyInAnyOrder(expectedTestCaseOutputList);
+  }
+
+  // Test that a class that does not have any Reporter output does not add any system-out elements.
+  @Test
+  public void ensureTestReportContainsNoSysOutContent() throws Exception {
+    Class<?> testClass = SimpleTestSample.class;
+    Path outputDir = TestHelper.createRandomDirectory();
+    TestNG tng = createTests(outputDir, "suite", testClass);
+    tng.setUseDefaultListeners(true);
+    Reporter.clear();
+    tng.run();
+    Document doc = getJunitReport(outputDir, testClass);
+    assertThat(doc.getElementsByTagName("system-out").getLength()).isEqualTo(0);
+  }
+
+  private Document getJunitReport(Path outputDir, Class<?> testClass)
+      throws IOException, ParserConfigurationException, SAXException {
     DocumentBuilder builder = getJUnitDocumentBuilder();
-    String name = "TEST-" + TestClassSample.class.getName();
+    String name = "TEST-" + testClass.getName();
     File file =
         new File(
             outputDir.toFile().getAbsolutePath()
@@ -145,11 +303,7 @@ public void ensureTestReportContainsValidSysOutContent() throws Exception {
                 + File.separator
                 + name
                 + ".xml");
-    Document doc = builder.parse(file);
-    XPath xPath = XPathFactory.newInstance().newXPath();
-    String expression = "//testsuite/system-out";
-    String data = (String) xPath.compile(expression).evaluate(doc, XPathConstants.STRING);
-    assertThat(data.trim()).isEqualTo(TestClassSample.MESSAGE_1 + "\n" + TestClassSample.MESSAGE_2);
+    return builder.parse(file);
   }
 
   private DocumentBuilder getJUnitDocumentBuilder()

testng-core/src/test/java/test/junitreports/issue2124/TestClassSample.java

@@ -1,16 +1,69 @@
 package test.junitreports.issue2124;
 
+import org.testng.Assert;
 import org.testng.Reporter;
+import org.testng.annotations.AfterSuite;
+import org.testng.annotations.BeforeSuite;
+import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
 public class TestClassSample {
 
   public static final String MESSAGE_1 = "Teenage Mutant Ninja Turtles";
   public static final String MESSAGE_2 = "Teenage Mutant Ninja Turtles: <i>Out of the Shadows</i>";
+  public static final String MESSAGE_3 =
+      "Teenage Mutant Ninja Turtles: <i>The Secret of the Ooze</i>";
+  public static final String MESSAGE_4 = "Teenage Mutant Ninja Turtles: <i>Mutant Mayhem</i>";
+  public static final String MESSAGE_5 =
+      "Teenage Mutant Ninja Turtles: <i>Rise of the Teenage Mutant Ninja Turtles</i>";
+  public static final String MESSAGE_BEFORE = "Teenage Mutant Ninja Turtles Movies";
+  public static final String MESSAGE_AFTER = "To be continued";
+  public static final String MESSAGE_FAIL = "Cowabunga";
+
+  public static final String TEST_METHOD_WITH_REPORTER = "testReporter";
+  public static final String TEST_METHOD_WITH_DATA_PROVIDER_REPORTER =
+      "testReporterWithDataProvider";
+  public static final String TEST_METHOD_FAIL_WITH_REPORTER = "testFailWithReporter";
+  public static final String TEST_METHOD_FAIL_NO_REPORTER = "testFailNoReporter";
+  public static final String TEST_METHOD_NO_REPORTER = "testNoReporter";
+
+  @BeforeSuite
+  public void before() {
+    Reporter.log(MESSAGE_BEFORE, true);
+  }
+
+  @AfterSuite
+  public void after() {
+    Reporter.log(MESSAGE_AFTER, true);
+  }
 
   @Test
   public void testReporter() {
     Reporter.log(MESSAGE_1, true);
     Reporter.log(MESSAGE_2, true);
   }
+
+  @DataProvider(name = "testReporterDataProvider")
+  public Object[][] testReporterDataProvider() {
+    return new Object[][] {{MESSAGE_3}, {MESSAGE_4}};
+  }
+
+  @Test(dataProvider = "testReporterDataProvider")
+  public void testReporterWithDataProvider(String message) {
+    Reporter.log(message, true);
+  }
+
+  @Test
+  public void testFailWithReporter() {
+    Reporter.log(MESSAGE_5, true);
+    Assert.fail(MESSAGE_FAIL);
+  }
+
+  @Test
+  public void testFailNoReporter() {
+    Assert.fail(MESSAGE_FAIL);
+  }
+
+  @Test
+  public void testNoReporter() {}
 }