From b53928e83649c6b11b93178c4d17afc02fa7982e Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Sat, 22 Apr 2023 10:41:59 -0700 Subject: [PATCH] Have event processor get configs from raw githubusercontent instead of needing a sparse checkout (#6030) * Have event processor fetch config files * Minor change to method name and add descriptions --- .../CodeOwnersParser/FileHelpers.cs | 39 +++++++++++++++++-- .../MockGitHubEventClient.cs | 4 +- .../Constants/ConfigConstants.cs | 14 +++++++ .../GitHubEventClient.cs | 38 ++++++++++++++++-- .../Program.cs | 5 +++ .../Utils/CodeOwnerUtils.cs | 4 +- .../Utils/RulesConfiguration.cs | 22 ++++++++--- 7 files changed, 110 insertions(+), 16 deletions(-) create mode 100644 tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/Constants/ConfigConstants.cs diff --git a/tools/code-owners-parser/CodeOwnersParser/FileHelpers.cs b/tools/code-owners-parser/CodeOwnersParser/FileHelpers.cs index 91db19aa3e1..8cf238a9224 100644 --- a/tools/code-owners-parser/CodeOwnersParser/FileHelpers.cs +++ b/tools/code-owners-parser/CodeOwnersParser/FileHelpers.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.Net; using System.Net.Http; namespace Azure.Sdk.Tools.CodeOwnersParser @@ -22,10 +23,42 @@ public static string GetFileOrUrlContents(string fileOrUrl) private static string GetUrlContents(string url) { + int maxRetries = 3; + int attempts = 1; + int delayTimeInMs = 1000; using HttpClient client = new HttpClient(); - HttpResponseMessage response = - client.GetAsync(url).ConfigureAwait(false).GetAwaiter().GetResult(); - return response.Content.ReadAsStringAsync().ConfigureAwait(false).GetAwaiter().GetResult(); + while (attempts <= maxRetries) + { + try + { + HttpResponseMessage response = client.GetAsync(url).ConfigureAwait(false).GetAwaiter().GetResult(); + if (response.StatusCode == HttpStatusCode.OK) + { + // This writeline is probably unnecessary but good to have if there are previous attempts that failed + Console.WriteLine($"GetUrlContents for {url} attempt number {attempts} succeeded."); + return response.Content.ReadAsStringAsync().ConfigureAwait(false).GetAwaiter().GetResult(); + } + else + { + Console.WriteLine($"GetUrlContents attempt number {attempts}. Non-{HttpStatusCode.OK} status code trying to fetch {url}. Status Code = {response.StatusCode}"); + } + } + catch (HttpRequestException httpReqEx) + { + // HttpRequestException means the request failed due to an underlying issue such as network connectivity, + // DNS failure, server certificate validation or timeout. + Console.WriteLine($"GetUrlContents attempt number {attempts}. HttpRequestException trying to fetch {url}. Exception message = {httpReqEx.Message}"); + if (attempts == maxRetries) + { + // At this point the retries have been exhausted, let this rethrow + throw; + } + } + System.Threading.Thread.Sleep(delayTimeInMs); + attempts++; + } + // This will only get hit if the final retry is non-OK status code + throw new FileLoadException($"Unable to fetch {url} after {maxRetries}. See above for status codes for each attempt."); } } } diff --git a/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/MockGitHubEventClient.cs b/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/MockGitHubEventClient.cs index ef99656cd8d..e113c0623e9 100644 --- a/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/MockGitHubEventClient.cs +++ b/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor.Tests/MockGitHubEventClient.cs @@ -39,8 +39,8 @@ public class MockGitHubEventClient: GitHubEventClient public SearchIssuesResult SearchIssuesResultReturn { get; set; } = new SearchIssuesResult(); - public MockGitHubEventClient(string productHeaderName, string? rulesConfigLocation = null) : - base(productHeaderName, rulesConfigLocation) + public MockGitHubEventClient(string productHeaderName) : + base(productHeaderName) { } diff --git a/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/Constants/ConfigConstants.cs b/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/Constants/ConfigConstants.cs new file mode 100644 index 00000000000..6c00a18661d --- /dev/null +++ b/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/Constants/ConfigConstants.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Azure.Sdk.Tools.GitHubEventProcessor.Constants +{ + public class ConfigConstants + { + public const string RawGitHubUserContentUrl = "https://raw.githubusercontent.com"; + public const string DefaultBranch = "main"; + } +} diff --git a/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/GitHubEventClient.cs b/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/GitHubEventClient.cs index 3294a8d5d91..e3dfd03f78c 100644 --- a/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/GitHubEventClient.cs +++ b/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/GitHubEventClient.cs @@ -131,13 +131,18 @@ public GitHubIssueToUpdate(long repositoryId, int issueOrPRNumber, IssueUpdate i public RulesConfiguration RulesConfiguration { - get { return _rulesConfiguration; } + get { + if (null == _rulesConfiguration) + { + _rulesConfiguration = LoadRulesConfiguration(); + } + return _rulesConfiguration; + } } - public GitHubEventClient(string productHeaderName, string rulesConfigLocation = null) + public GitHubEventClient(string productHeaderName) { _gitHubClient = CreateClientWithGitHubEnvToken(productHeaderName); - _rulesConfiguration = LoadRulesConfiguration(rulesConfigLocation); } /// @@ -940,5 +945,32 @@ public virtual async Task> QueryAILabelService(IssueEventGitHubPayl } return returnList; } + + /// + /// Create a raw github URL (https://raw.githubusercontent.com) for a file in a given repository + /// + /// Octkit.Repository from the event payload + /// Subdirectory where the file lives + /// name of the file + /// + public string CreateRawGitHubURLForFile(Repository repository, string subdirectory, string fileName) + { + // https://raw.githubusercontent.com/Azure/azure-sdk-for-net/main/.github/ + // The Full URL is BaseUrl + repositoryFullName + defaultBranch + remoteFilePath + fileName + string fileUrl = $"{ConfigConstants.RawGitHubUserContentUrl}/{repository.FullName}/{ConfigConstants.DefaultBranch}/{subdirectory}/{fileName}"; + return fileUrl; + } + + /// + /// Set the config file overrides for codeowners and rulesconfig which will cause them to get pulled + /// from the URL instead of requiring a sparse checkout of the configuration directory in order to run. + /// + /// Octkit.Repository from the event payload + public void SetConfigEntryOverrides(Repository repository) + { + CodeOwnerUtils.codeOwnersFilePathOverride = CreateRawGitHubURLForFile(repository, CodeOwnerUtils.CodeownersSubDirectory, CodeOwnerUtils.CodeownersFileName); + RulesConfiguration.rulesConfigFilePathOverride = CreateRawGitHubURLForFile(repository, RulesConfiguration.RulesConfigSubDirectory, RulesConfiguration.RulesConfigFileName); + } + } } diff --git a/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/Program.cs b/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/Program.cs index ffdbfc43233..459484dd637 100644 --- a/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/Program.cs +++ b/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/Program.cs @@ -41,12 +41,14 @@ static async Task Main(string[] args) case EventConstants.Issues: { IssueEventGitHubPayload issueEventPayload = serializer.Deserialize(rawJson); + gitHubEventClient.SetConfigEntryOverrides(issueEventPayload.Repository); await IssueProcessing.ProcessIssueEvent(gitHubEventClient, issueEventPayload); break; } case EventConstants.IssueComment: { IssueCommentPayload issueCommentPayload = serializer.Deserialize(rawJson); + gitHubEventClient.SetConfigEntryOverrides(issueCommentPayload.Repository); // IssueComment events are for both issues and pull requests. If the comment is on a pull request, // then Issue's PullRequest object in the payload will be non-null if (issueCommentPayload.Issue.PullRequest != null) @@ -65,12 +67,14 @@ static async Task Main(string[] args) // The pull_request, because of the auto_merge processing, requires more than just deserialization of the // the rawJson. PullRequestEventGitHubPayload prEventPayload = PullRequestProcessing.DeserializePullRequest(rawJson, serializer); + gitHubEventClient.SetConfigEntryOverrides(prEventPayload.Repository); await PullRequestProcessing.ProcessPullRequestEvent(gitHubEventClient, prEventPayload); break; } case EventConstants.PullRequestReview: { PullRequestReviewEventPayload prReviewEventPayload = serializer.Deserialize(rawJson); + gitHubEventClient.SetConfigEntryOverrides(prReviewEventPayload.Repository); await PullRequestReviewProcessing.ProcessPullRequestReviewEvent(gitHubEventClient, prReviewEventPayload); break; } @@ -87,6 +91,7 @@ static async Task Main(string[] args) } ScheduledEventGitHubPayload scheduledEventPayload = serializer.Deserialize(rawJson); + gitHubEventClient.SetConfigEntryOverrides(scheduledEventPayload.Repository); string cronTaskToRun = args[2]; await ScheduledEventProcessing.ProcessScheduledEvent(gitHubEventClient, scheduledEventPayload, cronTaskToRun); break; diff --git a/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/Utils/CodeOwnerUtils.cs b/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/Utils/CodeOwnerUtils.cs index 5bb3c2272a8..440fa4358f1 100644 --- a/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/Utils/CodeOwnerUtils.cs +++ b/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/Utils/CodeOwnerUtils.cs @@ -13,8 +13,8 @@ namespace Azure.Sdk.Tools.GitHubEventProcessor.Utils /// public class CodeOwnerUtils { - private static readonly string CodeownersFileName = "CODEOWNERS"; - private static readonly string CodeownersSubDirectory = ".github"; + public static readonly string CodeownersFileName = "CODEOWNERS"; + public static readonly string CodeownersSubDirectory = ".github"; static List _codeOwnerEntries = null; public static string codeOwnersFilePathOverride = null; diff --git a/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/Utils/RulesConfiguration.cs b/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/Utils/RulesConfiguration.cs index da1b854b2e7..55397935d03 100644 --- a/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/Utils/RulesConfiguration.cs +++ b/tools/github-event-processor/Azure.Sdk.Tools.GitHubEventProcessor/Utils/RulesConfiguration.cs @@ -6,6 +6,7 @@ using System.Text.Json.Serialization; using Azure.Sdk.Tools.GitHubEventProcessor.Constants; using System.IO; +using Azure.Sdk.Tools.CodeOwnersParser; namespace Azure.Sdk.Tools.GitHubEventProcessor.Utils { @@ -27,8 +28,9 @@ public enum RuleState /// public class RulesConfiguration { - private static readonly string RulesConfigFileName = "event-processor.config"; - private static readonly string RulesConfigSubDirectory = ".github"; + public static readonly string RulesConfigFileName = "event-processor.config"; + public static readonly string RulesConfigSubDirectory = ".github"; + public static string rulesConfigFilePathOverride = null; public Dictionary Rules { get; set; } public string RulesConfigFile { get; set; } = null; public RulesConfiguration() @@ -47,9 +49,17 @@ public RulesConfiguration(string configurationFile = null) string configLoc = configurationFile; if (configLoc == null) { - // Load the config from the well known location, somewhere under the .github directory - // which is in the root of the repository - configLoc = DirectoryUtils.FindFileInRepository(RulesConfigFileName, RulesConfigSubDirectory); + if (null != rulesConfigFilePathOverride) + { + // If the user overrode the location + configLoc = rulesConfigFilePathOverride; + } + else + { + // Load the config from the well known location, somewhere under the .github directory + // which is in the root of the repository + configLoc = DirectoryUtils.FindFileInRepository(RulesConfigFileName, RulesConfigSubDirectory); + } } RulesConfigFile = configLoc; LoadRulesFromConfig(); @@ -63,8 +73,8 @@ public void LoadRulesFromConfig() { if (null != RulesConfigFile) { + string rawJson = FileHelpers.GetFileOrUrlContents(RulesConfigFile); Console.WriteLine($"Loading repository rules from {RulesConfigFile}"); - string rawJson = File.ReadAllText(RulesConfigFile); Rules = JsonSerializer.Deserialize>(rawJson); // Report any rules that might be missing from the config file. ReportMissingRulesFromConfig();