Reporting changes (#7646)

Reintroduce corrected #7383 This switches arcade from using a custom reporter, to using a reporter more tightly integrated with helix. That reporter has the capacity to handle local test retrying and reporting to Azure DevOps for "flaky" tests. To enable the retry, a file "eng/test-configuration.json" must be created following the documented schema.
dotnet · Aug 10, 2021 · b4fd1cc · b4fd1cc
1 parent 4e176af
commit b4fd1cc
Show file tree

Hide file tree

Showing 16 changed files with 119 additions and 617 deletions.
diff --git a/.gitignore b/.gitignore
@@ -131,3 +131,9 @@ node_modules/
 # Python Compile Outputs
 
 *.pyc
+
+# IntelliJ
+.idea/
+
+# vscode python env files
+.env
diff --git a/eng/test-configuration.json b/eng/test-configuration.json
@@ -0,0 +1,5 @@
+{
+  "version": 1,
+  "defaultOnFailure": "rerun",
+  "localRerunCount": 1
+}
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelpersTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelpersTests.cs
@@ -1,5 +1,8 @@
+using System;
+using System.IO;
 using Xunit;
 using System.Net;
+using Newtonsoft.Json;
 
 namespace Microsoft.DotNet.Helix.Sdk.Tests
 {
@@ -35,5 +38,18 @@ public void VerifyNonEncodedFowardSlashIsConverted()
 
             Assert.Equal(workItemNameExpected, actual);
         }
+
+        [Fact]
+        public void FailOnceThenPass()
+        {
+            string target = Path.Combine(Environment.GetEnvironmentVariable("HELIX_WORKITEM_ROOT") ?? Environment.GetEnvironmentVariable("TEMP"), "my-test-file-123456.snt");
+            bool exists = File.Exists(target);
+            if (!exists)
+            {
+                File.WriteAllText(target, "Test failed once");
+            }
+
+            Assert.True(exists, $"File should exist: {target}");
+        }
     }
 }
diff --git a/src/Microsoft.DotNet.Helix/Sdk/SendHelixJob.cs b/src/Microsoft.DotNet.Helix/Sdk/SendHelixJob.cs
@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Globalization;
 using System.IO;
+using System.IO.Compression;
 using System.Linq;
 using System.Text;
 using System.Threading;
@@ -33,6 +34,7 @@ public static class MetadataNames
             public const string Uri = "Uri";
             public const string Destination = "Destination";
             public const string IncludeDirectoryName = "IncludeDirectoryName";
+            public const string AsArchive = "AsArchive";
         }
 
         /// <summary>
@@ -521,16 +523,43 @@ private IJobDefinition AddCorrelationPayload(IJobDefinition def, ITaskItem corre
             if (Directory.Exists(path))
             {
                 string includeDirectoryNameStr = correlationPayload.GetMetadata(MetadataNames.IncludeDirectoryName);
-                bool.TryParse(includeDirectoryNameStr, out bool includeDirectoryName);
+                if (!bool.TryParse(includeDirectoryNameStr, out bool includeDirectoryName))
+                {
+                    includeDirectoryName = false;
+                }
 
-                Log.LogMessage(MessageImportance.Low, $"Adding Correlation Payload Directory '{path}', destination '{destination}'");
+                Log.LogMessage(
+                    MessageImportance.Low,
+                    $"Adding Correlation Payload Directory '{path}', destination '{destination}'"
+                );
                 return def.WithCorrelationPayloadDirectory(path, includeDirectoryName, destination);
+
             }
 
             if (File.Exists(path))
             {
-                Log.LogMessage(MessageImportance.Low, $"Adding Correlation Payload Archive '{path}', destination '{destination}'");
-                return def.WithCorrelationPayloadArchive(path, destination);
+                string asArchiveStr = correlationPayload.GetMetadata(MetadataNames.AsArchive);
+                if (!bool.TryParse(asArchiveStr, out bool asArchive))
+                {
+                    // With no other information, default to true, since that was the previous behavior
+                    // before we added the option
+                    asArchive = true;
+                }
+
+                if (asArchive)
+                {
+                    Log.LogMessage(
+                        MessageImportance.Low,
+                        $"Adding Correlation Payload Archive '{path}', destination '{destination}'"
+                    );
+                    return def.WithCorrelationPayloadArchive(path, destination);
+                }
+
+                Log.LogMessage(
+                    MessageImportance.Low,
+                    $"Adding Correlation Payload File '{path}', destination '{destination}'"
+                );
+                return def.WithCorrelationPayloadFiles(path);
             }
 
             Log.LogError(FailureCategory.Build, $"Correlation Payload '{path}' not found.");

diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.MonoQueue.targets b/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.MonoQueue.targets
@@ -3,7 +3,11 @@
   <PropertyGroup>
     <EnableXUnitReporter Condition=" '$(EnableXUnitReporter)' != 'true' ">false</EnableXUnitReporter>
   </PropertyGroup>
-
+
+  <PropertyGroup Condition=" '$(HelixTestConfigurationFilePath)' == '' ">
+    <HelixTestConfigurationFilePath Condition=" '$(RepositoryEngineeringDir)' != '' ">$(RepositoryEngineeringDir)/test-configuration.json</HelixTestConfigurationFilePath>
+  </PropertyGroup>
+
   <Choose>
     <When Condition="$(HelixTargetQueue.ToLowerInvariant().Contains('windows'))">
       <PropertyGroup>
@@ -36,6 +40,10 @@
     <HelixProperties Condition="'$(HelixArchitecture)' != ''" Include="architecture" Value="$(HelixArchitecture)" />
     <HelixProperties Include="operatingSystem" Value="$(HelixTargetQueue)" />
   </ItemGroup>
+
+  <ItemGroup Condition="Exists('$(HelixTestConfigurationFilePath)')">
+    <HelixCorrelationPayload Include="$(HelixTestConfigurationFilePath)" AsArchive="false" />
+  </ItemGroup>
 
   <Target Name="CoreBuild">
   </Target>

diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/AzurePipelines.MonoQueue.targets b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/AzurePipelines.MonoQueue.targets
@@ -7,11 +7,11 @@
   <PropertyGroup>
     <HelixPostCommands Condition="$(IsPosixShell)">
       $(HelixPostCommands);
-      /bin/sh $HELIX_CORRELATION_PAYLOAD/reporter/run.sh $(SYSTEM_TEAMFOUNDATIONCOLLECTIONURI) $(SYSTEM_TEAMPROJECT) $(TestRunId) $(SYSTEM_ACCESSTOKEN) || exit $?
+      $HELIX_PYTHONPATH $HELIX_CORRELATION_PAYLOAD/reporter/run.py $(SYSTEM_TEAMFOUNDATIONCOLLECTIONURI) $(SYSTEM_TEAMPROJECT) $(TestRunId) $(SYSTEM_ACCESSTOKEN) || exit $?
     </HelixPostCommands>
     <HelixPostCommands Condition="!$(IsPosixShell)">
       $(HelixPostCommands);
-      call %HELIX_CORRELATION_PAYLOAD%\reporter\run.bat $(SYSTEM_TEAMFOUNDATIONCOLLECTIONURI) $(SYSTEM_TEAMPROJECT) $(TestRunId) $(SYSTEM_ACCESSTOKEN) || exit /b
+      %HELIX_PYTHONPATH% %HELIX_CORRELATION_PAYLOAD%\reporter\run.py $(SYSTEM_TEAMFOUNDATIONCOLLECTIONURI) $(SYSTEM_TEAMPROJECT) $(TestRunId) $(SYSTEM_ACCESSTOKEN) || exit /b
     </HelixPostCommands>
   </PropertyGroup>