Merge pull request #27 from quarkus-qe/feature/add-github-pr-commentator

Add JBang script that will crunch Flaky Run reports and create comment that can be printed in GitHub PRs where flakiness occurred

.github/workflows/release.yml

@@ -48,10 +48,11 @@ jobs:
           git checkout ${{github.base_ref}}
           git rebase release
           mvn -B release:perform -DskipITs -DskipTests -Prelease,framework -s maven-settings.xml
-      - name: Bump dependency version used in JBang script to the released version
+      - name: Bump dependency version used in JBang scripts to the released version
         run: |
           sed -i "s#DEPS io.quarkus.qe:flaky-run-reporter:.*#DEPS io.quarkus.qe:flaky-run-reporter:${{steps.metadata.outputs.current-version}}#" ./jbang-scripts/FlakyTestRunSummarizer.java
-          git commit -am "Update JBang script dependency version to ${{steps.metadata.outputs.current-version}}"
+          sed -i "s#DEPS io.quarkus.qe:flaky-run-reporter:.*#DEPS io.quarkus.qe:flaky-run-reporter:${{steps.metadata.outputs.current-version}}#" ./jbang-scripts/GitHubPrCommentator.java
+          git commit -am "Update JBang scripts dependency version to ${{steps.metadata.outputs.current-version}}"
       - name: Push changes to ${{github.base_ref}}
         uses: ad-m/[email protected]
         with:

jbang-scripts/GitHubPrCommentator.java

@@ -0,0 +1,16 @@
+//usr/bin/env jbang "$0" "$@" ; exit $?
+
+//DEPS io.quarkus.qe:flaky-run-reporter:0.1.2.Beta1
+
+import io.quarkus.qe.reporter.flakyrun.commentator.CreateGhPrComment;
+
+public class GitHubPrCommentator {
+    public static void main(String... args) {
+        try {
+            new CreateGhPrComment(args).printToStdOut();
+            System.exit(0);
+        } catch (Exception e) {
+            System.exit(1);
+        }
+    }
+}

src/main/java/io/quarkus/qe/reporter/flakyrun/FlakyReporterUtils.java

@@ -0,0 +1,58 @@
+package io.quarkus.qe.reporter.flakyrun;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+public final class FlakyReporterUtils {
+
+    private static final String EQUALS = "=";
+
+    private FlakyReporterUtils() {
+    }
+
+    public static boolean isArgument(String argumentKey, String argument) {
+        return argument.startsWith(argumentKey + EQUALS);
+    }
+
+    public static int parseIntArgument(String argumentKey, String argument) {
+        return Integer.parseInt(argument.substring((argumentKey + EQUALS).length()));
+    }
+
+    public static String parseStringArgument(String argumentKey, String argument) {
+        return argument.substring((argumentKey + EQUALS).length());
+    }
+
+    public static String getRequiredArgument(String argumentKey, String[] arguments) {
+        String argument = null;
+        for (String a : arguments) {
+            if (a != null && isArgument(argumentKey, a)) {
+                argument = a;
+            }
+        }
+        if (argument == null) {
+            throw new IllegalArgumentException("Argument '" + argument + "' is missing");
+        }
+        return parseStringArgument(argumentKey, argument);
+    }
+
+    public static String readFile(Path overviewPath) {
+        try {
+            return Files.readString(overviewPath);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public static String[] createCommandArgs(String... args) {
+        if (args.length % 2 != 0) {
+            throw new IllegalArgumentException("Args must be even");
+        }
+        String[] result = new String[args.length / 2];
+        for (int i = 0; i < result.length; i++) {
+            int j = i * 2;
+            result[i] = args[j] + EQUALS + args[j + 1];
+        }
+        return result;
+    }
+}

src/main/java/io/quarkus/qe/reporter/flakyrun/commentator/CreateGhPrComment.java

@@ -0,0 +1,89 @@
+package io.quarkus.qe.reporter.flakyrun.commentator;
+
+import io.quarkus.qe.reporter.flakyrun.reporter.FlakyRunReporter;
+import io.quarkus.qe.reporter.flakyrun.reporter.FlakyTest;
+import io.quarkus.qe.reporter.flakyrun.summary.FlakyRunSummaryReporter;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import static io.quarkus.qe.reporter.flakyrun.FlakyReporterUtils.getRequiredArgument;
+import static io.quarkus.qe.reporter.flakyrun.FlakyReporterUtils.readFile;
+
+/**
+ * This class is used by a Jbang script to simplify commenting in GitHub PRs when a flake were detected.
+ */
+public final class CreateGhPrComment {
+
+    public static final String TEST_BASE_DIR = CreateGhPrComment.class.getSimpleName() + ".test-base-dir";
+    public static final String OVERVIEW_FILE_KEY = "overview-file";
+    public static final String FLAKY_REPORTS_FILE_PREFIX_KEY = "flaky-reports-file-prefix";
+    private static final Path CURRENT_DIR = Path.of(".");
+    private final String comment;
+    private final Path baseDir;
+
+    public CreateGhPrComment(String[] args) {
+        if (System.getProperty(TEST_BASE_DIR) != null) {
+            baseDir = Path.of(System.getProperty(TEST_BASE_DIR));
+        } else {
+            baseDir = CURRENT_DIR;
+        }
+        var failureOverview = getFailureOverview(args);
+        var flakyTestsReports = getFlakyTestReports(args);
+        this.comment = """
+                Following jobs contain at least one flaky test: %s
+
+                %s
+                """.formatted(failureOverview, flakyTestsReports);
+    }
+
+    public String getComment() {
+        return comment;
+    }
+
+    public void printToStdOut() {
+        System.out.println(comment);
+    }
+
+    private String getFlakyTestReports(String[] args) {
+        var reportFilePrefix = getRequiredArgument(FLAKY_REPORTS_FILE_PREFIX_KEY, args);
+        var listOfDirFiles = baseDir.toFile().listFiles();
+        if (listOfDirFiles == null || listOfDirFiles.length == 0) {
+            return "No flaky test reports found";
+        }
+        var result = new StringBuilder();
+        for (File file : listOfDirFiles) {
+            if (file.getName().startsWith(reportFilePrefix)) {
+                var flakyTests = FlakyRunReporter.parseFlakyTestsReport(file.toPath());
+                if (!flakyTests.isEmpty()) {
+                    result.append("**Artifact `%s` contains following failures:**".formatted(file.getName()));
+                    for (FlakyTest flakyTest : flakyTests) {
+                        result.append("""
+
+                                - Test name: `%s`
+                                  Date and time: %s
+                                  Failure message: `%s`
+                                  Failure stacktrace:
+                                ```
+                                %s
+                                ```
+                                """.formatted(flakyTest.fullTestName(), flakyTest.dateTime(),
+                                flakyTest.failureMessage(), flakyTest.failureStackTrace()));
+                    }
+                    result.append(System.lineSeparator());
+                }
+            }
+        }
+        return result.toString();
+    }
+
+    private String getFailureOverview(String[] args) {
+        var overviewPath = baseDir.resolve(getRequiredArgument(OVERVIEW_FILE_KEY, args));
+        if (Files.exists(overviewPath)) {
+            return readFile(overviewPath);
+        }
+        throw new IllegalStateException("File '" + overviewPath + "' not found");
+    }
+}

src/main/java/io/quarkus/qe/reporter/flakyrun/summary/FlakyRunSummaryReporter.java

@@ -17,6 +17,9 @@
 import java.util.List;
 import java.util.stream.Collectors;
 
+import static io.quarkus.qe.reporter.flakyrun.FlakyReporterUtils.isArgument;
+import static io.quarkus.qe.reporter.flakyrun.FlakyReporterUtils.parseIntArgument;
+import static io.quarkus.qe.reporter.flakyrun.FlakyReporterUtils.parseStringArgument;
 import static java.util.stream.Collectors.groupingBy;
 
 public class FlakyRunSummaryReporter {
@@ -30,7 +33,6 @@ public class FlakyRunSummaryReporter {
     private static final String PREVIOUS_SUMMARY_REPORT_PATH = "previous-summary-report-path";
     private static final String NEW_SUMMARY_REPORT_PATH = "new-summary-report-path";
     private static final String NEW_FLAKY_REPORT_PATH = "new-flaky-report-path";
-    private static final String EQUALS = "=";
     private static final String CI_JOB_NAME = "flaky-report-ci-job-name";
     private final int dayRetention;
     private final int maxFlakesPerTest;
@@ -202,16 +204,4 @@ private static FlakyRunSummary parsePreviousSummary(Path summaryPath) {
         }
         return null;
     }
-
-    private static boolean isArgument(String argumentKey, String argument) {
-        return argument.startsWith(argumentKey + EQUALS);
-    }
-
-    private static int parseIntArgument(String argumentKey, String argument) {
-        return Integer.parseInt(argument.substring((argumentKey + EQUALS).length()));
-    }
-
-    private static String parseStringArgument(String argumentKey, String argument) {
-        return argument.substring((argumentKey + EQUALS).length());
-    }
 }

src/test/java/io/quarkus/qe/reporter/flakyrun/FlakyRunReportTest.java

@@ -1,5 +1,6 @@
 package io.quarkus.qe.reporter.flakyrun;
 
+import io.quarkus.qe.reporter.flakyrun.commentator.CreateGhPrComment;
 import io.quarkus.qe.reporter.flakyrun.summary.FlakyRunSummaryReporter;
 import org.apache.maven.shared.invoker.DefaultInvocationRequest;
 import org.apache.maven.shared.invoker.DefaultInvoker;
@@ -21,11 +22,14 @@
 import java.util.Objects;
 import java.util.Properties;
 
+import static io.quarkus.qe.reporter.flakyrun.FlakyReporterUtils.createCommandArgs;
+import static io.quarkus.qe.reporter.flakyrun.FlakyReporterUtils.readFile;
+import static io.quarkus.qe.reporter.flakyrun.commentator.CreateGhPrComment.FLAKY_REPORTS_FILE_PREFIX_KEY;
+import static io.quarkus.qe.reporter.flakyrun.commentator.CreateGhPrComment.OVERVIEW_FILE_KEY;
 import static io.quarkus.qe.reporter.flakyrun.reporter.FlakyRunReporter.FLAKY_RUN_REPORT;
 import static io.quarkus.qe.reporter.flakyrun.summary.FlakyRunSummaryReporter.CI_BUILD_NUMBER;
 import static io.quarkus.qe.reporter.flakyrun.summary.FlakyRunSummaryReporter.DAY_RETENTION;
 import static io.quarkus.qe.reporter.flakyrun.summary.FlakyRunSummaryReporter.FLAKY_SUMMARY_REPORT;
-import static io.quarkus.qe.reporter.flakyrun.summary.FlakyRunSummaryReporter.TEST_BASE_DIR;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -54,9 +58,47 @@ public void test() throws IOException, MavenInvocationException {
 
         assertGeneratedFlakyRunReport();
         assertFlakyRunSummary();
+        assertGitHubPrCommentator();
     }
 
-    private void assertFlakyRunSummary() throws IOException {
+    private static void assertGitHubPrCommentator() throws IOException {
+        var testTarget = getFlakyRunReportFile().toPath().getParent();
+        System.setProperty(CreateGhPrComment.TEST_BASE_DIR, testTarget.toString());
+
+        // prepare overview
+        var overview = new File("src/test/resources/overview_file.txt");
+        var overviewInTestTarget = new File(TARGET_FLAKY_TEST_DIR, "target/" + overview.getName());
+        FileUtils.copyFile(overview, overviewInTestTarget);
+        assertFalse(readFile(overviewInTestTarget.toPath()).isBlank());
+
+        // prepare flaky run reports
+        var expectedReportPrefix = "flaky-run-report-";
+        var newFlakyRunReportFile1 = testTarget.resolve(expectedReportPrefix + "whatever-1").toFile();
+        var newFlakyRunReportFile2 = testTarget.resolve(expectedReportPrefix + "whatever-2").toFile();
+        FileUtils.copyFile(getFlakyRunReportFile(), newFlakyRunReportFile1);
+        FileUtils.copyFile(getFlakyRunReportFile(), newFlakyRunReportFile2);
+
+        // prepare comment
+        var commentator = new CreateGhPrComment(createCommandArgs(OVERVIEW_FILE_KEY, overview.getName(),
+                FLAKY_REPORTS_FILE_PREFIX_KEY, expectedReportPrefix));
+        var comment = commentator.getComment();
+
+        // assert comment
+        assertTrue(
+                comment.contains(
+                        "Artifact `%s` contains following failures:".formatted(newFlakyRunReportFile1.getName())),
+                comment);
+        assertTrue(
+                comment.contains(
+                        "Artifact `%s` contains following failures:".formatted(newFlakyRunReportFile2.getName())),
+                comment);
+        assertTrue(comment.contains("Failure message: `failing to test flakiness reporting`"), comment);
+        assertTrue(comment.contains("Failure stacktrace:"), comment);
+        assertTrue(comment.contains("org.opentest4j.AssertionFailedError: failing to test flakiness reporting"),
+                comment);
+    }
+
+    private static void assertFlakyRunSummary() throws IOException {
         // use old summary I downloaded from Jenkins, if format changes, it needs to change as well
         var oldSummary = new File("src/test/resources/flaky-summary-report.json");
         var summaryTarget = new File(TARGET_FLAKY_TEST_DIR, "target/" + FLAKY_SUMMARY_REPORT);
@@ -65,11 +107,11 @@ private void assertFlakyRunSummary() throws IOException {
         assertTrue(previousValue.contains("PicocliDevIT.verifyGreetingCommandOutputsExpectedMessage"), previousValue);
         assertFalse(previousValue.contains("FlakyTest.testFlaky"), previousValue);
 
-        System.setProperty(TEST_BASE_DIR, getFlakyRunReportFile().getParent());
+        System.setProperty(FlakyRunSummaryReporter.TEST_BASE_DIR, getFlakyRunReportFile().getParent());
         // making it maximal day retention because the old message needs to be valid for this test to pass
         var expectedBuildNumber = "987654321";
         new FlakyRunSummaryReporter(
-                new String[] { CI_BUILD_NUMBER + "=" + expectedBuildNumber, DAY_RETENTION + "=" + Integer.MAX_VALUE })
+                createCommandArgs(CI_BUILD_NUMBER, expectedBuildNumber, DAY_RETENTION, Integer.MAX_VALUE + ""))
                         .createReport();
 
         // now assert the old summary and new flaky run report were merged

src/test/resources/overview_file.txt

@@ -0,0 +1 @@
+'PR - Linux - JVM build - Latest Version', 'PR - Linux - Native build - Latest Version', 'PR - Windows - JVM build - Latest Version'