feat: Make reporting of code coverage more consistent (#152)

Adjust how code coverage reports are generated and make behavior consistent between local and CI builds

- The combined coverage data (Cobertura.xml) will be placed directly in the code coverage output directory
- The "Report" directory will always contain a HTML report (no longer a Azure Pipelines specific report when running on Azure Pipelines.
- The HTML report as well as the Cobertura.xml is now published as pipeline artifact on both Azure Pipelines and GitHub Actions
- The Azure Pipelines-specific HTML report that is shown in the web UI is now generated in a temporary directory (and only when publishing results to Azure Pipelines)

Pull-Request: #152

src/SharedBuild/Tasks/TestTask.cs

@@ -1,6 +1,5 @@
 using System;
 using System.Collections.Generic;
-using System.Diagnostics;
 using System.Linq;
 using System.Threading.Tasks;
 using Cake.Common.Build;
@@ -9,9 +8,11 @@
 using Cake.Common.Tools.DotNet;
 using Cake.Common.Tools.DotNet.Test;
 using Cake.Common.Tools.ReportGenerator;
+using Cake.Core;
 using Cake.Core.Diagnostics;
 using Cake.Core.IO;
 using Cake.Frosting;
+using Grynwald.SharedBuild.Tools;
 using Grynwald.SharedBuild.Tools.TemporaryFiles;
 
 namespace Grynwald.SharedBuild.Tasks;
@@ -28,7 +29,7 @@ public override async Task RunAsync(IBuildContext context)
 
         if (context.TestSettings.CollectCodeCoverage)
         {
-            await GenerateCoverageReportAsync(context);
+            await GenerateCodeCoverageOutputAsync(context);
         }
     }
 
@@ -136,108 +137,172 @@ await context.GitHubActions().Commands.UploadArtifact(
         }
     }
 
-    private async Task GenerateCoverageReportAsync(IBuildContext context)
+    protected virtual IReadOnlyDictionary<FilePath, string> GetTestRunNames(IBuildContext context, IEnumerable<FilePath> testResultPaths)
+    {
+        var testRunNamer = new TestRunNamer(context.Log, context.Environment, context.FileSystem);
+
+        var previousNames = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+        var testRunNames = new Dictionary<FilePath, string>();
+
+        foreach (var testResultPath in testResultPaths)
+        {
+            var baseName = testRunNamer.GetTestRunName(testResultPath);
+            var name = baseName;
+
+            // Test run names should be unique, otherwise Azure Pipeline will overwrite results for a previous test with the same name
+            // To avoid this, append a number at the end of the name until it is unique.
+            var counter = 1;
+            while (previousNames.Contains(name))
+            {
+                name = $"{baseName} ({counter++})";
+            }
+
+            previousNames.Add(name);
+            testRunNames.Add(testResultPath, name);
+        }
+
+        return testRunNames;
+    }
+
+    /// <summary>
+    /// Merges the individual code coverage reports from all test projects into a single coverage result file and generates a HTML report.
+    /// If the build is running in a CI system, the coverage is also published as pipeline artifact
+    /// </summary>
+    private async Task GenerateCodeCoverageOutputAsync(IBuildContext context)
     {
-        context.EnsureDirectoryDoesNotExist(context.Output.CodeCoverageReportDirectory, new() { Force = true, Recursive = true });
+        var mergedCoverageResult = MergeCoverageFiles(context);
+
+        var htmlReportPath = GenerateCodeCoverageHtmlReport(context, mergedCoverageResult);
 
+        //
+        // Publish Code coverage report to CI artifacts if necessary
+        //
+        if (context.AzurePipelines.IsActive)
+        {
+            PublishCodeCoverageToAzurePipelines(context, mergedCoverageResult, htmlReportPath);
+        }
+        else if (context.GitHubActions.IsActive)
+        {
+            await PublishCodeCoverageToGitHubActionsAsync(context, mergedCoverageResult, htmlReportPath);
+        }
+    }
+
+    /// <summary>
+    /// Merges all code coverage outputs into a single Cobertura report file
+    /// </summary>
+    protected virtual FilePath MergeCoverageFiles(IBuildContext context)
+    {
         var coverageFiles = context.FileSystem.GetFilePaths(context.Output.TestResultsDirectory, "coverage.cobertura.xml", SearchScope.Recursive);
 
         if (!coverageFiles.Any())
             throw new Exception($"No coverage files found in '{context.Output.TestResultsDirectory}'");
 
         context.Log.Information($"Found {coverageFiles.Count} coverage files");
 
+        var mergedCoverageFilePath = context.Output.CodeCoverageOutputDirectory.CombineWithFilePath("Cobertura.xml");
+        context.EnsureFileDoesNotExist(mergedCoverageFilePath);
+
         //
-        // Generate Coverage Report and merged code coverage file
+        // Generate merged code coverage file
         //
         context.Log.Information("Merging coverage files");
-        var htmlReportType = context.AzurePipelines.IsActive
-            ? ReportGeneratorReportType.HtmlInline_AzurePipelines
-            : ReportGeneratorReportType.Html;
-
         context.ReportGenerator(
             reports: coverageFiles,
-            targetDir: context.Output.CodeCoverageReportDirectory,
+            targetDir: context.Output.CodeCoverageOutputDirectory,
             settings: new ReportGeneratorSettings()
             {
-                ReportTypes = [htmlReportType, ReportGeneratorReportType.Cobertura],
-                HistoryDirectory = context.Output.CodeCoverageHistoryDirectory
+                ReportTypes = [ReportGeneratorReportType.Cobertura],
+                HistoryDirectory = GetCodeCoverageHistoryDirectory(context)
             }
         );
 
-        var coverageReportPath = context.Output.CodeCoverageReportDirectory.CombineWithFilePath("Cobertura.xml");
-
-        //
-        // Publish Code coverage report
-        //
-        if (context.AzurePipelines.IsActive)
+        if (!context.FileExists(mergedCoverageFilePath))
         {
-            PublishCodeCoverageToAzurePipelines(context, coverageReportPath);
+            throw new CakeException($"Failed to merge code coverage output files. Expected output file '{mergedCoverageFilePath}' does not exist after merging files");
         }
-        else if (context.GitHubActions.IsActive)
-        {
-            await PublishCodeCoverageToGitHubActionsAsync(context, coverageReportPath);
-        }
-    }
 
-    protected virtual void PublishCodeCoverageToAzurePipelines(IBuildContext context, FilePath coverageReportPath)
-    {
-        context.Log.Information("Publishing Code Coverage Results to Azure Pipelines");
-        context.AzurePipelines.Commands.PublishCodeCoverage(new()
-        {
-            CodeCoverageTool = AzurePipelinesCodeCoverageToolType.Cobertura,
-            SummaryFileLocation = coverageReportPath,
-            ReportDirectory = context.Output.CodeCoverageReportDirectory
-        });
+        return mergedCoverageFilePath;
     }
 
-    protected virtual async Task PublishCodeCoverageToGitHubActionsAsync(IBuildContext context, FilePath coverageReportPath)
+    /// <summary>
+    /// Generates a Code Coverage HTML report
+    /// </summary>
+    protected virtual DirectoryPath GenerateCodeCoverageHtmlReport(IBuildContext context, FilePath coverageFilePath)
     {
-        context.Log.Information("Publishing Code Coverage Results to GitHub Actions");
-
-        using var temporaryDirectory = context.CreateTemporaryDirectory();
+        var reportDirectory = context.Output.CodeCoverageOutputDirectory.Combine("Report");
+        context.Log.Information($"Generating code coverage HTML report to '{reportDirectory}'");
+        context.EnsureDirectoryDoesNotExist(reportDirectory);
 
-        // Generate Markdown coverage report
         context.ReportGenerator(
-            reports: [coverageReportPath],
-            targetDir: temporaryDirectory.Path.Combine("Report"),
+            reports: [coverageFilePath],
+            targetDir: reportDirectory,
             settings: new ReportGeneratorSettings()
             {
                 ReportTypes = [ReportGeneratorReportType.Html],
-                HistoryDirectory = context.Output.CodeCoverageHistoryDirectory,
+                HistoryDirectory = GetCodeCoverageHistoryDirectory(context)
             }
         );
 
-        context.CopyFileToDirectory(coverageReportPath, temporaryDirectory.Path);
-
-        // Publish coverage file and Summary as artifacts
-        await context.GitHubActions().Commands.UploadArtifact(temporaryDirectory.Path, "CodeCoverage");
+        return reportDirectory;
     }
 
-    protected virtual IReadOnlyDictionary<FilePath, string> GetTestRunNames(IBuildContext context, IEnumerable<FilePath> testResultPaths)
-    {
-        var testRunNamer = new TestRunNamer(context.Log, context.Environment, context.FileSystem);
+    private DirectoryPath GetCodeCoverageHistoryDirectory(IBuildContext context) => context.Output.CodeCoverageOutputDirectory.Combine("History");
 
-        var previousNames = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
-        var testRunNames = new Dictionary<FilePath, string>();
 
-        foreach (var testResultPath in testResultPaths)
-        {
-            var baseName = testRunNamer.GetTestRunName(testResultPath);
-            var name = baseName;
+    protected virtual void PublishCodeCoverageToAzurePipelines(IBuildContext context, FilePath coverageReportPath, DirectoryPath htmlReportPath)
+    {
+        context.Log.Information("Publishing Code Coverage Results to Azure Pipelines");
 
-            // Test run names should be unique, otherwise Azure Pipeline will overwrite results for a previous test with the same name
-            // To avoid this, append a number at the end of the name until it is unique.
-            var counter = 1;
-            while (previousNames.Contains(name))
+        //
+        // Generate a version of the HTML report tailored for Azure Pipelines and publish it as code coverage
+        // so it is shown in the "Code Coverage" Azure Pipelines Web UI
+        //
+        var azurePipelinesHtmlReportDirectory = context.AzurePipelines.Environment.Build.ArtifactStagingDirectory.Combine($"{Guid.NewGuid():n}");
+        context.Log.Verbose("Generating tailored HTML code coverage report for Azure Pipelines");
+        context.ReportGenerator(
+            reports: [coverageReportPath],
+            targetDir: azurePipelinesHtmlReportDirectory,
+            settings: new ReportGeneratorSettings()
             {
-                name = $"{baseName} ({counter++})";
+                ReportTypes = [ReportGeneratorReportType.HtmlInline_AzurePipelines],
+                HistoryDirectory = GetCodeCoverageHistoryDirectory(context)
             }
+        );
 
-            previousNames.Add(name);
-            testRunNames.Add(testResultPath, name);
-        }
+        context.Log.Verbose("Publishing code coverage to Azure Pipelines Web UI");
+        context.AzurePipelines.Commands.PublishCodeCoverage(new()
+        {
+            CodeCoverageTool = AzurePipelinesCodeCoverageToolType.Cobertura,
+            SummaryFileLocation = coverageReportPath,
+            ReportDirectory = azurePipelinesHtmlReportDirectory
+        });
 
-        return testRunNames;
+        //
+        // Publish HTML report and coverage report as pipeline artifact to make it downloadable
+        //
+
+        var artifactStagingDirectory = context.AzurePipelines.Environment.Build.ArtifactStagingDirectory.Combine("CodeCoverage");
+        context.EnsureDirectoryDoesNotExist(artifactStagingDirectory);
+        context.EnsureDirectoryExists(artifactStagingDirectory);
+
+        context.CopyFileToDirectory(coverageReportPath, artifactStagingDirectory);
+        context.CopyDirectory(htmlReportPath, artifactStagingDirectory.Combine(htmlReportPath.GetDirectoryName()));
+        context.Log.Verbose($"Publishing code coverage as pipeline artifact");
+        context.AzurePipelines.Commands.UploadArtifact("", artifactStagingDirectory.ToString(), "CodeCoverage");
+    }
+
+    protected virtual async Task PublishCodeCoverageToGitHubActionsAsync(IBuildContext context, FilePath coverageReportPath, DirectoryPath htmlReportPath)
+    {
+        context.Log.Information("Publishing Code Coverage Results to GitHub Actions");
+
+        // GitHub Actions only allows artifact uploads once for each name
+        // => Copy all files together into a temporary directory and publish that directory
+        using var temporaryDirectory = context.CreateTemporaryDirectory();
+
+        context.CopyFileToDirectory(coverageReportPath, temporaryDirectory.Path);
+        context.CopyDirectory(htmlReportPath, temporaryDirectory.Path.Combine(htmlReportPath.GetDirectoryName()));
+
+        // Publish coverage file and Summary as artifacts
+        await context.GitHubActions().Commands.UploadArtifact(temporaryDirectory.Path, "CodeCoverage");
     }
 }

src/SharedBuild/Tools/FileAliases.cs

@@ -0,0 +1,16 @@
+using Cake.Core;
+using Cake.Core.IO;
+
+namespace Grynwald.SharedBuild.Tools;
+
+public static class FileAliases
+{
+    public static void EnsureFileDoesNotExist(this ICakeContext context, FilePath path)
+    {
+        var file = context.FileSystem.GetFile(path);
+        if (file.Exists)
+        {
+            file.Delete();
+        }
+    }
+}

src/SharedBuild/_Context/IOutputContext.cs

@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using Cake.Core.IO;
 
 namespace Grynwald.SharedBuild;
@@ -26,15 +27,9 @@ public interface IOutputContext : IPrintableObject
     DirectoryPath TestResultsDirectory { get; }
 
     /// <summary>
-    /// Gets the output path for code coverage reports
+    /// Gets the output path for the code coverage report
     /// </summary>
-    DirectoryPath CodeCoverageReportDirectory { get; }
-
-    /// <summary>
-    /// Gets the output path for code coverage history files
-    /// (used by Report Generator to show differences in code coverage between different runs)
-    /// </summary>
-    DirectoryPath CodeCoverageHistoryDirectory { get; }
+    DirectoryPath CodeCoverageOutputDirectory { get; }
 
     /// <summary>
     /// Gets all NuGet package files in the packages output directory

src/SharedBuild/_Context/_Default/DefaultOutputContext.cs

@@ -28,10 +28,9 @@ public virtual DirectoryPath BinariesDirectory
     public virtual DirectoryPath TestResultsDirectory => BinariesDirectory.Combine(m_Context.BuildSettings.Configuration).Combine("TestResults");
 
     /// <inheritdoc />
-    public virtual DirectoryPath CodeCoverageReportDirectory => BinariesDirectory.Combine(m_Context.BuildSettings.Configuration).Combine("CodeCoverage").Combine("Report");
+    public virtual DirectoryPath CodeCoverageOutputDirectory => BinariesDirectory.Combine(m_Context.BuildSettings.Configuration).Combine("CodeCoverage");
+
 
-    /// <inheritdoc />
-    public virtual DirectoryPath CodeCoverageHistoryDirectory => BinariesDirectory.Combine(m_Context.BuildSettings.Configuration).Combine("CodeCoverage").Combine("History");
 
     /// <inheritdoc />
     public virtual FilePath ChangeLogFile => BinariesDirectory.CombineWithFilePath("changelog.md");
@@ -46,8 +45,7 @@ public virtual void PrintToLog(ICakeLog log)
         log.Information($"{nameof(BinariesDirectory)}: {BinariesDirectory.FullPath}");
         log.Information($"{nameof(PackagesDirectory)}: {PackagesDirectory.FullPath}");
         log.Information($"{nameof(TestResultsDirectory)}: {TestResultsDirectory.FullPath}");
-        log.Information($"{nameof(CodeCoverageReportDirectory)}: {CodeCoverageReportDirectory.FullPath}");
-        log.Information($"{nameof(CodeCoverageHistoryDirectory)}: {CodeCoverageHistoryDirectory.FullPath}");
+        log.Information($"{nameof(CodeCoverageOutputDirectory)}: {CodeCoverageOutputDirectory.FullPath}");
         log.Information($"{nameof(ChangeLogFile)}: {ChangeLogFile.FullPath}");
     }
 }