Add test coverage for Flaky Run Report

.github/workflows/build.yml

@@ -1,4 +1,4 @@
-name: Build
+name: Build and Test this PR
 
 on:
   push:
@@ -44,7 +44,16 @@ jobs:
         key: maven-repo-${{ steps.get-date.outputs.date }}
 
     - name: Build with Maven
-      run: mvn -B formatter:validate verify --file pom.xml
+      run: mvn -B formatter:validate clean install -DskipTests -DskipITs --file pom.xml
+
+    - name: Find and store Maven home
+      id: maven-home-discovery
+      run: echo "detected-mvn-home=$(mvn -v | grep 'Maven home:' | sed -e 's|Maven home:| |' | xargs)" >> "$GITHUB_OUTPUT"
+
+    - name: Run tests
+      run: mvn clean verify
+      env:
+        MAVEN_HOME: ${{ steps.maven-home-discovery.outputs.detected-mvn-home }}
 
     - name: Delete Artifacts From Cache
       shell: bash

.github/workflows/release.yml

@@ -47,7 +47,7 @@ jobs:
           mvn -B release:prepare -Prelease,framework,extensions,examples -DautoVersionSubmodules -DreleaseVersion=${{steps.metadata.outputs.current-version}} -DdevelopmentVersion=${{steps.metadata.outputs.next-version}} -s maven-settings.xml
           git checkout ${{github.base_ref}}
           git rebase release
-          mvn -B release:perform -DskipITs -Prelease,framework -s maven-settings.xml
+          mvn -B release:perform -DskipITs -DskipTests -Prelease,framework -s maven-settings.xml
       - name: Bump dependency version used in JBang script to the released version
         run: |
           sed -i "s#DEPS io.quarkus.qe:flaky-run-reporter:.*#DEPS io.quarkus.qe:flaky-run-reporter:${{steps.metadata.outputs.current-version}}#" ./jbang-scripts/FlakyTestRunSummarizer.java

pom.xml

@@ -20,6 +20,10 @@
         <formatter-maven-plugin.version>2.24.1</formatter-maven-plugin.version>
         <!-- Format Settings -->
         <src.format.goal>format</src.format.goal>
+        <!-- Test Settings -->
+        <junit.jupiter.version>5.11.0</junit.jupiter.version>
+        <maven-invoker.version>3.3.0</maven-invoker.version>
+        <maven.surefire.version>3.5.0</maven.surefire.version>
     </properties>
 
     <dependencies>
@@ -43,13 +47,27 @@
             <artifactId>surefire-report-parser</artifactId>
             <version>${maven-surefire-report-parser.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.maven.shared</groupId>
+            <artifactId>maven-invoker</artifactId>
+            <version>${maven-invoker.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter</artifactId>
+            <version>${junit.jupiter.version}</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <build>
         <plugins>
             <plugin>
                 <groupId>org.codehaus.plexus</groupId>
                 <artifactId>plexus-component-metadata</artifactId>
+                <!-- FIXME: replace this deprecated dependency -->
+                <version>2.2.0</version>
                 <executions>
                     <execution>
                         <goals>
@@ -73,6 +91,19 @@
                     <lineEnding>LF</lineEnding>
                 </configuration>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <version>${maven.surefire.version}</version>
+                <configuration>
+                    <systemPropertyVariables>
+                        <!-- so that we test same setup (deps) and self in the Maven Invoker test -->
+                        <junit.jupiter.version>${junit.jupiter.version}</junit.jupiter.version>
+                        <maven.surefire.version>${maven.surefire.version}</maven.surefire.version>
+                        <flaky-run-reporter.version>${project.version}</flaky-run-reporter.version>
+                    </systemPropertyVariables>
+                </configuration>
+            </plugin>
         </plugins>
     </build>
 

src/main/java/io/quarkus/qe/reporter/flakyrun/mavenextension/FlakyRunReportingMavenExtension.java

@@ -19,16 +19,27 @@ public class FlakyRunReportingMavenExtension extends AbstractMavenLifecycleParti
 
     @Override
     public void afterSessionEnd(MavenSession session) {
+        logger.debug("Flaky run reporter started");
+
         var projects = getProjectsFromMvnSession(session);
         if (!projects.isEmpty()) {
             new FlakyRunReporter(logger).createReport(projects);
+        } else {
+            logger.info("No projects found in this Maven session, won't generate Flaky Run report");
         }
     }
 
     private static List<Project> getProjectsFromMvnSession(MavenSession session) {
         final Path rootPath = Path.of("").toAbsolutePath();
+
+        // for multi-module projects, we don't inspect project root, because there are no tests
+        // but for single-module project we should just check 'target' of that project
+        boolean isNotMultiModuleProject = session.getResult().getTopologicallySortedProjects().size() == 1;
+
         return session.getResult().getTopologicallySortedProjects().stream()
-                .map(p -> new Project(p.getName(), rootPath.relativize(p.getBasedir().toPath())))
-                .filter(p -> !p.baseDir().toString().isEmpty()).toList();
+                .map(p -> new Project(p.getName(), rootPath.relativize(p.getBasedir().toPath()))).filter(p -> {
+                    var rootProject = !p.baseDir().toString().isEmpty();
+                    return isNotMultiModuleProject || rootProject;
+                }).toList();
     }
 }

src/test/java/io/quarkus/qe/reporter/flakyrun/FlakyRunReportTest.java

@@ -0,0 +1,109 @@
+package io.quarkus.qe.reporter.flakyrun;
+
+import org.apache.maven.shared.invoker.DefaultInvocationRequest;
+import org.apache.maven.shared.invoker.DefaultInvoker;
+import org.apache.maven.shared.invoker.InvocationRequest;
+import org.apache.maven.shared.invoker.MavenInvocationException;
+import org.apache.maven.shared.invoker.PrintStreamHandler;
+import org.apache.maven.shared.invoker.SystemOutHandler;
+import org.codehaus.plexus.util.FileUtils;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.nio.file.Files;
+import java.util.List;
+import java.util.Objects;
+import java.util.Properties;
+
+import static io.quarkus.qe.reporter.flakyrun.reporter.FlakyRunReporter.FLAKY_RUN_REPORT;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class FlakyRunReportTest {
+
+    private static final File TEMPLATE_FLAKY_TEST_DIR = new File("src/test/resources-flaky-test");
+    private static final File TARGET_FLAKY_TEST_DIR = new File("target/flaky-test");
+
+    // HINT: always build this project first and then run tests
+    // otherwise in my experience, you are testing previous version
+    // maybe because result of the build is only stored to the local .m2 after all tests are run as well
+    @Test
+    public void test() throws IOException, MavenInvocationException {
+        var invoker = new DefaultInvoker();
+        FileUtils.copyDirectoryStructure(TEMPLATE_FLAKY_TEST_DIR, TARGET_FLAKY_TEST_DIR);
+        invoker.setWorkingDirectory(TARGET_FLAKY_TEST_DIR);
+        invoker.setMavenHome(getMavenHome());
+        try (var request = mvnCleanTestRequest()) {
+            invoker.execute(request.delegate);
+            var runLogs = request.outputStream.toString();
+            assertTrue(runLogs.contains("Tests run: 1, Failures: 0, Errors: 0, Skipped: 0, Flakes: 1"));
+            assertTrue(runLogs.contains("Run 1: FlakyTest.testFlaky:18 failing to test flakiness reporting"));
+            assertTrue(runLogs.contains("Run 2: PASS"));
+            System.out.println(runLogs);
+        }
+
+        assertGeneratedFlakyRunReport();
+    }
+
+    private static void assertGeneratedFlakyRunReport() throws IOException {
+        var flakyRunReport = new File(TARGET_FLAKY_TEST_DIR, "target/" + FLAKY_RUN_REPORT);
+        assertTrue(flakyRunReport.exists(),
+                "Flaky Run report wasn't generated, '%s' does not exist".formatted(flakyRunReport));
+        var reportContent = Files.readString(flakyRunReport.toPath());
+        assertReportContent(reportContent, "\"projectName\" : \"Flaky Run Reporter - Failing Test\"");
+        assertReportContent(reportContent,
+                "\"fullTestName\" : \"io.quarkus.qe.reporter.flakyrun.FlakyTest.testFlaky\"");
+        assertReportContent(reportContent, "\"failureType\" : \"org.opentest4j.AssertionFailedError\"");
+        assertReportContent(reportContent, "failureStackTrace");
+        assertReportContent(reportContent, "FlakyTest.java:18"); // part of the stacktrace
+        assertReportContent(reportContent, "dateTime");
+    }
+
+    private static void assertReportContent(String reportContent, String expectedPortion) {
+        assertTrue(reportContent.contains(expectedPortion), reportContent);
+    }
+
+    private static File getMavenHome() {
+        var mvnHome = System.getenv("MAVEN_HOME");
+        Objects.requireNonNull(mvnHome, "Environment variable 'MAVEN_HOME' is required");
+        return new File(mvnHome);
+    }
+
+    private static CloseableRequest mvnCleanTestRequest() {
+        DefaultInvocationRequest request = new DefaultInvocationRequest();
+        request.setGoals(List.of("clean", "verify"));
+        request.setShowErrors(true);
+        request.setDebug(false);
+        request.setBaseDirectory(TARGET_FLAKY_TEST_DIR);
+        request.setPomFile(new File(TARGET_FLAKY_TEST_DIR, "pom.xml"));
+        request.setShellEnvironmentInherited(true);
+        request.setProperties(getProperties());
+        var outputStream = new ByteArrayOutputStream();
+        var inputStream = new PrintStream(outputStream);
+        request.setOutputHandler(new PrintStreamHandler(inputStream, true));
+        request.setErrorHandler(new SystemOutHandler());
+        return new CloseableRequest(request, outputStream, inputStream);
+    }
+
+    private static Properties getProperties() {
+        var properties = new Properties();
+        properties.setProperty("junit.jupiter.version", System.getProperty("junit.jupiter.version"));
+        properties.setProperty("maven.surefire.version", System.getProperty("maven.surefire.version"));
+        properties.setProperty("flaky-run-reporter.version", System.getProperty("flaky-run-reporter.version"));
+        return properties;
+    }
+
+    private record CloseableRequest(InvocationRequest delegate, ByteArrayOutputStream outputStream,
+            PrintStream printStream) implements Closeable {
+
+        @Override
+        public void close() throws IOException {
+            printStream.close();
+            outputStream.close();
+        }
+    }
+}

src/test/resources-flaky-test/pom.xml

@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>io.quarkus.qe</groupId>
+    <artifactId>flaky-run-reporter-failing-test</artifactId>
+    <name>Flaky Run Reporter - Failing Test</name>
+    <version>1.0.0-SNAPSHOT</version>
+
+    <properties>
+        <maven.compiler.source>17</maven.compiler.source>
+        <maven.compiler.target>17</maven.compiler.target>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter</artifactId>
+            <version>${junit.jupiter.version}</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <extensions>
+            <extension>
+                <groupId>io.quarkus.qe</groupId>
+                <artifactId>flaky-run-reporter</artifactId>
+                <version>${flaky-run-reporter.version}</version>
+            </extension>
+        </extensions>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <version>${maven.surefire.version}</version>
+                <configuration>
+                    <!-- required to test flaky run report -->
+                    <rerunFailingTestsCount>1</rerunFailingTestsCount>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-failsafe-plugin</artifactId>
+                <version>${maven.surefire.version}</version>
+                <configuration>
+                    <!-- required to test flaky run report -->
+                    <rerunFailingTestsCount>1</rerunFailingTestsCount>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+</project>

src/test/resources-flaky-test/src/test/java/io/quarkus/qe/reporter/flakyrun/FlakyTest.java

@@ -0,0 +1,26 @@
+package io.quarkus.qe.reporter.flakyrun;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+public class FlakyTest {
+
+    private static final Path ALREADY_FAILED_FILE = Path.of("target").resolve("already-failed-test");
+
+    @Test
+    public void testFlaky() throws IOException {
+        if (!Files.exists(ALREADY_FAILED_FILE)) {
+            touch(ALREADY_FAILED_FILE);
+            Assertions.fail("failing to test flakiness reporting");
+        }
+    }
+
+    private static boolean touch(Path filePath) throws IOException {
+        return filePath.toFile().createNewFile();
+    }
+
+}