Have event processor get configs from raw githubusercontent instead o…

…f needing a sparse checkout (#6030)

* Have event processor fetch config files

* Minor change to method name and add descriptions

tools/code-owners-parser/CodeOwnersParser/FileHelpers.cs

@@ -1,5 +1,6 @@
 using System;
 using System.IO;
+using System.Net;
 using System.Net.Http;
 
 namespace Azure.Sdk.Tools.CodeOwnersParser
@@ -22,10 +23,42 @@ public static string GetFileOrUrlContents(string fileOrUrl)
 
         private static string GetUrlContents(string url)
         {
+            int maxRetries = 3;
+            int attempts = 1;
+            int delayTimeInMs = 1000;
             using HttpClient client = new HttpClient();
-            HttpResponseMessage response =
-                client.GetAsync(url).ConfigureAwait(false).GetAwaiter().GetResult();
-            return response.Content.ReadAsStringAsync().ConfigureAwait(false).GetAwaiter().GetResult();
+            while (attempts <= maxRetries)
+            {
+                try
+                {
+                    HttpResponseMessage response = client.GetAsync(url).ConfigureAwait(false).GetAwaiter().GetResult();
+                    if (response.StatusCode == HttpStatusCode.OK)
+                    {
+                        // This writeline is probably unnecessary but good to have if there are previous attempts that failed
+                        Console.WriteLine($"GetUrlContents for {url} attempt number {attempts} succeeded.");
+                        return response.Content.ReadAsStringAsync().ConfigureAwait(false).GetAwaiter().GetResult();
+                    }
+                    else
+                    {
+                        Console.WriteLine($"GetUrlContents attempt number {attempts}. Non-{HttpStatusCode.OK} status code trying to fetch {url}. Status Code = {response.StatusCode}");
+                    }
+                }
+                catch (HttpRequestException httpReqEx)
+                {
+                    // HttpRequestException means the request failed due to an underlying issue such as network connectivity,
+                    // DNS failure, server certificate validation or timeout.
+                    Console.WriteLine($"GetUrlContents attempt number {attempts}. HttpRequestException trying to fetch {url}. Exception message = {httpReqEx.Message}");
+                    if (attempts == maxRetries)
+                    {
+                        // At this point the retries have been exhausted, let this rethrow
+                        throw;
+                    }
+                }
+                System.Threading.Thread.Sleep(delayTimeInMs);
+                attempts++;
+            }
+            // This will only get hit if the final retry is non-OK status code
+            throw new FileLoadException($"Unable to fetch {url} after {maxRetries}. See above for status codes for each attempt.");
         }
     }
 }

tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/MockGitHubEventClient.cs

@@ -39,8 +39,8 @@ public class MockGitHubEventClient: GitHubEventClient
 
         public SearchIssuesResult SearchIssuesResultReturn { get; set; } = new SearchIssuesResult();
 
-        public MockGitHubEventClient(string productHeaderName, string? rulesConfigLocation = null) : 
-            base(productHeaderName, rulesConfigLocation)
+        public MockGitHubEventClient(string productHeaderName) : 
+            base(productHeaderName)
         {
 
         }

tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/Constants/ConfigConstants.cs

@@ -0,0 +1,14 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Azure.Sdk.Tools.GitHubEventProcessor.Constants
+{
+    public class ConfigConstants
+    {
+        public const string RawGitHubUserContentUrl = "https://raw.githubusercontent.com";
+        public const string DefaultBranch = "main";
+    }
+}

tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/GitHubEventClient.cs

@@ -131,13 +131,18 @@ public GitHubIssueToUpdate(long repositoryId, int issueOrPRNumber, IssueUpdate i
 
         public RulesConfiguration RulesConfiguration
         {
-            get { return _rulesConfiguration; }
+            get {
+                if (null == _rulesConfiguration) 
+                {
+                    _rulesConfiguration = LoadRulesConfiguration();
+                }
+                return _rulesConfiguration; 
+            }
         }
 
-        public GitHubEventClient(string productHeaderName, string rulesConfigLocation = null)
+        public GitHubEventClient(string productHeaderName)
         {
             _gitHubClient = CreateClientWithGitHubEnvToken(productHeaderName);
-            _rulesConfiguration = LoadRulesConfiguration(rulesConfigLocation);
         }
 
         /// <summary>
@@ -940,5 +945,32 @@ public virtual async Task<List<string>> QueryAILabelService(IssueEventGitHubPayl
             }
             return returnList;
         }
+
+        /// <summary>
+        /// Create a raw github URL (https://raw.githubusercontent.com) for a file in a given repository
+        /// </summary>
+        /// <param name="repository">Octkit.Repository from the event payload</param>
+        /// <param name="subdirectory">Subdirectory where the file lives</param>
+        /// <param name="fileName">name of the file</param>
+        /// <returns></returns>
+        public string CreateRawGitHubURLForFile(Repository repository, string subdirectory, string fileName)
+        {
+            // https://raw.githubusercontent.com/Azure/azure-sdk-for-net/main/.github/
+            // The Full URL is BaseUrl + repositoryFullName + defaultBranch + remoteFilePath + fileName
+            string fileUrl = $"{ConfigConstants.RawGitHubUserContentUrl}/{repository.FullName}/{ConfigConstants.DefaultBranch}/{subdirectory}/{fileName}";
+            return fileUrl;
+        }
+
+        /// <summary>
+        /// Set the config file overrides for codeowners and rulesconfig which will cause them to get pulled
+        /// from the URL instead of requiring a sparse checkout of the configuration directory in order to run.
+        /// </summary>
+        /// <param name="repository">Octkit.Repository from the event payload</param>
+        public void SetConfigEntryOverrides(Repository repository)
+        {
+            CodeOwnerUtils.codeOwnersFilePathOverride = CreateRawGitHubURLForFile(repository, CodeOwnerUtils.CodeownersSubDirectory, CodeOwnerUtils.CodeownersFileName);
+            RulesConfiguration.rulesConfigFilePathOverride = CreateRawGitHubURLForFile(repository, RulesConfiguration.RulesConfigSubDirectory, RulesConfiguration.RulesConfigFileName);
+        }
+
     }
 }

tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/Program.cs

@@ -41,12 +41,14 @@ static async Task Main(string[] args)
                 case EventConstants.Issues:
                     {
                         IssueEventGitHubPayload issueEventPayload = serializer.Deserialize<IssueEventGitHubPayload>(rawJson);
+                        gitHubEventClient.SetConfigEntryOverrides(issueEventPayload.Repository);
                         await IssueProcessing.ProcessIssueEvent(gitHubEventClient, issueEventPayload);
                         break;
                     }
                 case EventConstants.IssueComment:
                     {
                         IssueCommentPayload issueCommentPayload = serializer.Deserialize<IssueCommentPayload>(rawJson);
+                        gitHubEventClient.SetConfigEntryOverrides(issueCommentPayload.Repository);
                         // IssueComment events are for both issues and pull requests. If the comment is on a pull request,
                         // then Issue's PullRequest object in the payload will be non-null
                         if (issueCommentPayload.Issue.PullRequest != null)
@@ -65,12 +67,14 @@ static async Task Main(string[] args)
                         // The pull_request, because of the auto_merge processing, requires more than just deserialization of the
                         // the rawJson.
                         PullRequestEventGitHubPayload prEventPayload = PullRequestProcessing.DeserializePullRequest(rawJson, serializer);
+                        gitHubEventClient.SetConfigEntryOverrides(prEventPayload.Repository);
                         await PullRequestProcessing.ProcessPullRequestEvent(gitHubEventClient, prEventPayload);
                         break;
                     }
                 case EventConstants.PullRequestReview:
                     {
                         PullRequestReviewEventPayload prReviewEventPayload = serializer.Deserialize<PullRequestReviewEventPayload>(rawJson);
+                        gitHubEventClient.SetConfigEntryOverrides(prReviewEventPayload.Repository);
                         await PullRequestReviewProcessing.ProcessPullRequestReviewEvent(gitHubEventClient, prReviewEventPayload);
                         break;
                     }
@@ -87,6 +91,7 @@ static async Task Main(string[] args)
                         }
 
                         ScheduledEventGitHubPayload scheduledEventPayload = serializer.Deserialize<ScheduledEventGitHubPayload>(rawJson);
+                        gitHubEventClient.SetConfigEntryOverrides(scheduledEventPayload.Repository);
                         string cronTaskToRun = args[2];
                         await ScheduledEventProcessing.ProcessScheduledEvent(gitHubEventClient, scheduledEventPayload, cronTaskToRun);
                         break;

tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/Utils/CodeOwnerUtils.cs

@@ -13,8 +13,8 @@ namespace Azure.Sdk.Tools.GitHubEventProcessor.Utils
     /// </summary>
     public class CodeOwnerUtils
     {
-        private static readonly string CodeownersFileName = "CODEOWNERS";
-        private static readonly string CodeownersSubDirectory = ".github";
+        public static readonly string CodeownersFileName = "CODEOWNERS";
+        public static readonly string CodeownersSubDirectory = ".github";
 
         static List<CodeownersEntry> _codeOwnerEntries = null;
         public static string codeOwnersFilePathOverride = null;

tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/Utils/RulesConfiguration.cs

@@ -6,6 +6,7 @@
 using System.Text.Json.Serialization;
 using Azure.Sdk.Tools.GitHubEventProcessor.Constants;
 using System.IO;
+using Azure.Sdk.Tools.CodeOwnersParser;
 
 namespace Azure.Sdk.Tools.GitHubEventProcessor.Utils
 {
@@ -27,8 +28,9 @@ public enum RuleState
     /// </summary>
     public class RulesConfiguration
     {
-        private static readonly string RulesConfigFileName = "event-processor.config";
-        private static readonly string RulesConfigSubDirectory = ".github";
+        public static readonly string RulesConfigFileName = "event-processor.config";
+        public static readonly string RulesConfigSubDirectory = ".github";
+        public static string rulesConfigFilePathOverride = null;
         public Dictionary<string, RuleState> Rules { get; set; }
         public string RulesConfigFile { get; set; } = null;
         public RulesConfiguration() 
@@ -47,9 +49,17 @@ public RulesConfiguration(string configurationFile = null)
             string configLoc = configurationFile;
             if (configLoc == null)
             {
-                // Load the config from the well known location, somewhere under the .github directory
-                // which is in the root of the repository
-                configLoc = DirectoryUtils.FindFileInRepository(RulesConfigFileName, RulesConfigSubDirectory);
+                if (null != rulesConfigFilePathOverride)
+                {
+                    // If the user overrode the location
+                    configLoc = rulesConfigFilePathOverride;
+                }
+                else
+                {
+                    // Load the config from the well known location, somewhere under the .github directory
+                    // which is in the root of the repository
+                    configLoc = DirectoryUtils.FindFileInRepository(RulesConfigFileName, RulesConfigSubDirectory);
+                }
             }
             RulesConfigFile = configLoc;
             LoadRulesFromConfig();
@@ -63,8 +73,8 @@ public void LoadRulesFromConfig()
         {
             if (null != RulesConfigFile)
             {
+                string rawJson = FileHelpers.GetFileOrUrlContents(RulesConfigFile);
                 Console.WriteLine($"Loading repository rules from {RulesConfigFile}");
-                string rawJson = File.ReadAllText(RulesConfigFile);
                 Rules = JsonSerializer.Deserialize<Dictionary<string, RuleState>>(rawJson);
                 // Report any rules that might be missing from the config file.
                 ReportMissingRulesFromConfig();