From 3efca00f5e1c362a376a958d07739a0f0f8c0271 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Vav=C5=99=C3=ADk?= Date: Wed, 25 Sep 2024 18:38:32 +0200 Subject: [PATCH] Add JBang script for GH PR comments --- .github/workflows/release.yml | 5 +- jbang-scripts/GitHubPrCommentator.java | 16 ++++ .../reporter/flakyrun/FlakyReporterUtils.java | 58 ++++++++++++ .../commentator/CreateGhPrComment.java | 89 +++++++++++++++++++ .../summary/FlakyRunSummaryReporter.java | 16 +--- .../reporter/flakyrun/FlakyRunReportTest.java | 50 ++++++++++- src/test/resources/overview_file.txt | 1 + 7 files changed, 216 insertions(+), 19 deletions(-) create mode 100644 jbang-scripts/GitHubPrCommentator.java create mode 100644 src/main/java/io/quarkus/qe/reporter/flakyrun/FlakyReporterUtils.java create mode 100644 src/main/java/io/quarkus/qe/reporter/flakyrun/commentator/CreateGhPrComment.java create mode 100644 src/test/resources/overview_file.txt diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e39220e..363f94e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -48,10 +48,11 @@ jobs: git checkout ${{github.base_ref}} git rebase release mvn -B release:perform -DskipITs -DskipTests -Prelease,framework -s maven-settings.xml - - name: Bump dependency version used in JBang script to the released version + - name: Bump dependency version used in JBang scripts to the released version run: | sed -i "s#DEPS io.quarkus.qe:flaky-run-reporter:.*#DEPS io.quarkus.qe:flaky-run-reporter:${{steps.metadata.outputs.current-version}}#" ./jbang-scripts/FlakyTestRunSummarizer.java - git commit -am "Update JBang script dependency version to ${{steps.metadata.outputs.current-version}}" + sed -i "s#DEPS io.quarkus.qe:flaky-run-reporter:.*#DEPS io.quarkus.qe:flaky-run-reporter:${{steps.metadata.outputs.current-version}}#" ./jbang-scripts/GitHubPrCommentator.java + git commit -am "Update JBang scripts dependency version to ${{steps.metadata.outputs.current-version}}" - name: Push changes to ${{github.base_ref}} uses: ad-m/github-push-action@v0.8.0 with: diff --git a/jbang-scripts/GitHubPrCommentator.java b/jbang-scripts/GitHubPrCommentator.java new file mode 100644 index 0000000..1dd247a --- /dev/null +++ b/jbang-scripts/GitHubPrCommentator.java @@ -0,0 +1,16 @@ +//usr/bin/env jbang "$0" "$@" ; exit $? + +//DEPS io.quarkus.qe:flaky-run-reporter:0.1.2.Beta1 + +import io.quarkus.qe.reporter.flakyrun.commentator.CreateGhPrComment; + +public class GitHubPrCommentator { + public static void main(String... args) { + try { + new CreateGhPrComment(args).printToStdOut(); + System.exit(0); + } catch (Exception e) { + System.exit(1); + } + } +} diff --git a/src/main/java/io/quarkus/qe/reporter/flakyrun/FlakyReporterUtils.java b/src/main/java/io/quarkus/qe/reporter/flakyrun/FlakyReporterUtils.java new file mode 100644 index 0000000..b30eb0f --- /dev/null +++ b/src/main/java/io/quarkus/qe/reporter/flakyrun/FlakyReporterUtils.java @@ -0,0 +1,58 @@ +package io.quarkus.qe.reporter.flakyrun; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +public final class FlakyReporterUtils { + + private static final String EQUALS = "="; + + private FlakyReporterUtils() { + } + + public static boolean isArgument(String argumentKey, String argument) { + return argument.startsWith(argumentKey + EQUALS); + } + + public static int parseIntArgument(String argumentKey, String argument) { + return Integer.parseInt(argument.substring((argumentKey + EQUALS).length())); + } + + public static String parseStringArgument(String argumentKey, String argument) { + return argument.substring((argumentKey + EQUALS).length()); + } + + public static String getRequiredArgument(String argumentKey, String[] arguments) { + String argument = null; + for (String a : arguments) { + if (a != null && isArgument(argumentKey, a)) { + argument = a; + } + } + if (argument == null) { + throw new IllegalArgumentException("Argument '" + argument + "' is missing"); + } + return parseStringArgument(argumentKey, argument); + } + + public static String readFile(Path overviewPath) { + try { + return Files.readString(overviewPath); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static String[] createCommandArgs(String... args) { + if (args.length % 2 != 0) { + throw new IllegalArgumentException("Args must be even"); + } + String[] result = new String[args.length / 2]; + for (int i = 0; i < result.length; i++) { + int j = i * 2; + result[i] = args[j] + EQUALS + args[j + 1]; + } + return result; + } +} diff --git a/src/main/java/io/quarkus/qe/reporter/flakyrun/commentator/CreateGhPrComment.java b/src/main/java/io/quarkus/qe/reporter/flakyrun/commentator/CreateGhPrComment.java new file mode 100644 index 0000000..743ddea --- /dev/null +++ b/src/main/java/io/quarkus/qe/reporter/flakyrun/commentator/CreateGhPrComment.java @@ -0,0 +1,89 @@ +package io.quarkus.qe.reporter.flakyrun.commentator; + +import io.quarkus.qe.reporter.flakyrun.reporter.FlakyRunReporter; +import io.quarkus.qe.reporter.flakyrun.reporter.FlakyTest; +import io.quarkus.qe.reporter.flakyrun.summary.FlakyRunSummaryReporter; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import static io.quarkus.qe.reporter.flakyrun.FlakyReporterUtils.getRequiredArgument; +import static io.quarkus.qe.reporter.flakyrun.FlakyReporterUtils.readFile; + +/** + * This class is used by a Jbang script to simplify commenting in GitHub PRs when a flake were detected. + */ +public final class CreateGhPrComment { + + public static final String TEST_BASE_DIR = CreateGhPrComment.class.getSimpleName() + ".test-base-dir"; + public static final String OVERVIEW_FILE_KEY = "overview-file"; + public static final String FLAKY_REPORTS_FILE_PREFIX_KEY = "flaky-reports-file-prefix"; + private static final Path CURRENT_DIR = Path.of("."); + private final String comment; + private final Path baseDir; + + public CreateGhPrComment(String[] args) { + if (System.getProperty(TEST_BASE_DIR) != null) { + baseDir = Path.of(System.getProperty(TEST_BASE_DIR)); + } else { + baseDir = CURRENT_DIR; + } + var failureOverview = getFailureOverview(args); + var flakyTestsReports = getFlakyTestReports(args); + this.comment = """ + Following jobs contain at least one flaky test: %s + + %s + """.formatted(failureOverview, flakyTestsReports); + } + + public String getComment() { + return comment; + } + + public void printToStdOut() { + System.out.println(comment); + } + + private String getFlakyTestReports(String[] args) { + var reportFilePrefix = getRequiredArgument(FLAKY_REPORTS_FILE_PREFIX_KEY, args); + var listOfDirFiles = baseDir.toFile().listFiles(); + if (listOfDirFiles == null || listOfDirFiles.length == 0) { + return "No flaky test reports found"; + } + var result = new StringBuilder(); + for (File file : listOfDirFiles) { + if (file.getName().startsWith(reportFilePrefix)) { + var flakyTests = FlakyRunReporter.parseFlakyTestsReport(file.toPath()); + if (!flakyTests.isEmpty()) { + result.append("**Artifact `%s` contains following failures:**".formatted(file.getName())); + for (FlakyTest flakyTest : flakyTests) { + result.append(""" + + - Test name: `%s` + Date and time: %s + Failure message: `%s` + Failure stacktrace: + ``` + %s + ``` + """.formatted(flakyTest.fullTestName(), flakyTest.dateTime(), + flakyTest.failureMessage(), flakyTest.failureStackTrace())); + } + result.append(System.lineSeparator()); + } + } + } + return result.toString(); + } + + private String getFailureOverview(String[] args) { + var overviewPath = baseDir.resolve(getRequiredArgument(OVERVIEW_FILE_KEY, args)); + if (Files.exists(overviewPath)) { + return readFile(overviewPath); + } + throw new IllegalStateException("File '" + overviewPath + "' not found"); + } +} diff --git a/src/main/java/io/quarkus/qe/reporter/flakyrun/summary/FlakyRunSummaryReporter.java b/src/main/java/io/quarkus/qe/reporter/flakyrun/summary/FlakyRunSummaryReporter.java index a9aa04f..cdc84f3 100644 --- a/src/main/java/io/quarkus/qe/reporter/flakyrun/summary/FlakyRunSummaryReporter.java +++ b/src/main/java/io/quarkus/qe/reporter/flakyrun/summary/FlakyRunSummaryReporter.java @@ -17,6 +17,9 @@ import java.util.List; import java.util.stream.Collectors; +import static io.quarkus.qe.reporter.flakyrun.FlakyReporterUtils.isArgument; +import static io.quarkus.qe.reporter.flakyrun.FlakyReporterUtils.parseIntArgument; +import static io.quarkus.qe.reporter.flakyrun.FlakyReporterUtils.parseStringArgument; import static java.util.stream.Collectors.groupingBy; public class FlakyRunSummaryReporter { @@ -30,7 +33,6 @@ public class FlakyRunSummaryReporter { private static final String PREVIOUS_SUMMARY_REPORT_PATH = "previous-summary-report-path"; private static final String NEW_SUMMARY_REPORT_PATH = "new-summary-report-path"; private static final String NEW_FLAKY_REPORT_PATH = "new-flaky-report-path"; - private static final String EQUALS = "="; private static final String CI_JOB_NAME = "flaky-report-ci-job-name"; private final int dayRetention; private final int maxFlakesPerTest; @@ -202,16 +204,4 @@ private static FlakyRunSummary parsePreviousSummary(Path summaryPath) { } return null; } - - private static boolean isArgument(String argumentKey, String argument) { - return argument.startsWith(argumentKey + EQUALS); - } - - private static int parseIntArgument(String argumentKey, String argument) { - return Integer.parseInt(argument.substring((argumentKey + EQUALS).length())); - } - - private static String parseStringArgument(String argumentKey, String argument) { - return argument.substring((argumentKey + EQUALS).length()); - } } diff --git a/src/test/java/io/quarkus/qe/reporter/flakyrun/FlakyRunReportTest.java b/src/test/java/io/quarkus/qe/reporter/flakyrun/FlakyRunReportTest.java index 9af1b87..4c90f59 100644 --- a/src/test/java/io/quarkus/qe/reporter/flakyrun/FlakyRunReportTest.java +++ b/src/test/java/io/quarkus/qe/reporter/flakyrun/FlakyRunReportTest.java @@ -1,5 +1,6 @@ package io.quarkus.qe.reporter.flakyrun; +import io.quarkus.qe.reporter.flakyrun.commentator.CreateGhPrComment; import io.quarkus.qe.reporter.flakyrun.summary.FlakyRunSummaryReporter; import org.apache.maven.shared.invoker.DefaultInvocationRequest; import org.apache.maven.shared.invoker.DefaultInvoker; @@ -21,11 +22,14 @@ import java.util.Objects; import java.util.Properties; +import static io.quarkus.qe.reporter.flakyrun.FlakyReporterUtils.createCommandArgs; +import static io.quarkus.qe.reporter.flakyrun.FlakyReporterUtils.readFile; +import static io.quarkus.qe.reporter.flakyrun.commentator.CreateGhPrComment.FLAKY_REPORTS_FILE_PREFIX_KEY; +import static io.quarkus.qe.reporter.flakyrun.commentator.CreateGhPrComment.OVERVIEW_FILE_KEY; import static io.quarkus.qe.reporter.flakyrun.reporter.FlakyRunReporter.FLAKY_RUN_REPORT; import static io.quarkus.qe.reporter.flakyrun.summary.FlakyRunSummaryReporter.CI_BUILD_NUMBER; import static io.quarkus.qe.reporter.flakyrun.summary.FlakyRunSummaryReporter.DAY_RETENTION; import static io.quarkus.qe.reporter.flakyrun.summary.FlakyRunSummaryReporter.FLAKY_SUMMARY_REPORT; -import static io.quarkus.qe.reporter.flakyrun.summary.FlakyRunSummaryReporter.TEST_BASE_DIR; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -54,9 +58,47 @@ public void test() throws IOException, MavenInvocationException { assertGeneratedFlakyRunReport(); assertFlakyRunSummary(); + assertGitHubPrCommentator(); } - private void assertFlakyRunSummary() throws IOException { + private static void assertGitHubPrCommentator() throws IOException { + var testTarget = getFlakyRunReportFile().toPath().getParent(); + System.setProperty(CreateGhPrComment.TEST_BASE_DIR, testTarget.toString()); + + // prepare overview + var overview = new File("src/test/resources/overview_file.txt"); + var overviewInTestTarget = new File(TARGET_FLAKY_TEST_DIR, "target/" + overview.getName()); + FileUtils.copyFile(overview, overviewInTestTarget); + assertFalse(readFile(overviewInTestTarget.toPath()).isBlank()); + + // prepare flaky run reports + var expectedReportPrefix = "flaky-run-report-"; + var newFlakyRunReportFile1 = testTarget.resolve(expectedReportPrefix + "whatever-1").toFile(); + var newFlakyRunReportFile2 = testTarget.resolve(expectedReportPrefix + "whatever-2").toFile(); + FileUtils.copyFile(getFlakyRunReportFile(), newFlakyRunReportFile1); + FileUtils.copyFile(getFlakyRunReportFile(), newFlakyRunReportFile2); + + // prepare comment + var commentator = new CreateGhPrComment(createCommandArgs(OVERVIEW_FILE_KEY, overview.getName(), + FLAKY_REPORTS_FILE_PREFIX_KEY, expectedReportPrefix)); + var comment = commentator.getComment(); + + // assert comment + assertTrue( + comment.contains( + "Artifact `%s` contains following failures:".formatted(newFlakyRunReportFile1.getName())), + comment); + assertTrue( + comment.contains( + "Artifact `%s` contains following failures:".formatted(newFlakyRunReportFile2.getName())), + comment); + assertTrue(comment.contains("Failure message: `failing to test flakiness reporting`"), comment); + assertTrue(comment.contains("Failure stacktrace:"), comment); + assertTrue(comment.contains("org.opentest4j.AssertionFailedError: failing to test flakiness reporting"), + comment); + } + + private static void assertFlakyRunSummary() throws IOException { // use old summary I downloaded from Jenkins, if format changes, it needs to change as well var oldSummary = new File("src/test/resources/flaky-summary-report.json"); var summaryTarget = new File(TARGET_FLAKY_TEST_DIR, "target/" + FLAKY_SUMMARY_REPORT); @@ -65,11 +107,11 @@ private void assertFlakyRunSummary() throws IOException { assertTrue(previousValue.contains("PicocliDevIT.verifyGreetingCommandOutputsExpectedMessage"), previousValue); assertFalse(previousValue.contains("FlakyTest.testFlaky"), previousValue); - System.setProperty(TEST_BASE_DIR, getFlakyRunReportFile().getParent()); + System.setProperty(FlakyRunSummaryReporter.TEST_BASE_DIR, getFlakyRunReportFile().getParent()); // making it maximal day retention because the old message needs to be valid for this test to pass var expectedBuildNumber = "987654321"; new FlakyRunSummaryReporter( - new String[] { CI_BUILD_NUMBER + "=" + expectedBuildNumber, DAY_RETENTION + "=" + Integer.MAX_VALUE }) + createCommandArgs(CI_BUILD_NUMBER, expectedBuildNumber, DAY_RETENTION, Integer.MAX_VALUE + "")) .createReport(); // now assert the old summary and new flaky run report were merged diff --git a/src/test/resources/overview_file.txt b/src/test/resources/overview_file.txt new file mode 100644 index 0000000..0de8bb0 --- /dev/null +++ b/src/test/resources/overview_file.txt @@ -0,0 +1 @@ +'PR - Linux - JVM build - Latest Version', 'PR - Linux - Native build - Latest Version', 'PR - Windows - JVM build - Latest Version' \ No newline at end of file