From 6067f137ba8a8de2bcc1455e36d6769908347226 Mon Sep 17 00:00:00 2001 From: Konrad Jamrozik Date: Mon, 6 Feb 2023 15:00:07 -0800 Subject: [PATCH] fixups + refactor (#5352) --- .../PostConfigureKeyVaultSettings.cs | 3 +- .../Configuration/SecretClientProvider.cs | 4 +- .../Processor.cs | 96 ++++++++++++------- .../Program.cs | 6 +- .../PipelineOwnersExtractor.sln.DotSettings | 2 + 5 files changed, 70 insertions(+), 41 deletions(-) create mode 100644 tools/pipeline-owners-extractor/PipelineOwnersExtractor.sln.DotSettings diff --git a/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Configuration/PostConfigureKeyVaultSettings.cs b/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Configuration/PostConfigureKeyVaultSettings.cs index 1cb78569f71..38060890fef 100644 --- a/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Configuration/PostConfigureKeyVaultSettings.cs +++ b/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Configuration/PostConfigureKeyVaultSettings.cs @@ -1,7 +1,6 @@ -using System; +using System; using System.Linq; using System.Text.RegularExpressions; - using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; diff --git a/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Configuration/SecretClientProvider.cs b/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Configuration/SecretClientProvider.cs index 656c457a4f3..7cb566f8275 100644 --- a/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Configuration/SecretClientProvider.cs +++ b/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Configuration/SecretClientProvider.cs @@ -1,4 +1,4 @@ -using System; +using System; using Azure.Core; using Azure.Security.KeyVault.Secrets; @@ -19,4 +19,4 @@ public SecretClient GetSecretClient(Uri vaultUri) return new SecretClient(vaultUri, this.tokenCredential); } } -} \ No newline at end of file +} diff --git a/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Processor.cs b/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Processor.cs index 837a95b209e..d21baf5efc9 100644 --- a/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Processor.cs +++ b/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Processor.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -14,7 +14,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.TeamFoundation.Build.WebApi; - +using Models.OpenSourcePortal; using Newtonsoft.Json; namespace Azure.Sdk.Tools.PipelineOwnersExtractor @@ -46,38 +46,49 @@ public Processor( public async Task ExecuteAsync(CancellationToken stoppingToken = default) { - var projects = this.settings.Projects.Split(','); - var pipelineResults = await Task.WhenAll(projects.Select(x => devOpsService.GetPipelinesAsync(x.Trim()))); + string[] projects = this.settings.Projects.Split(','); + IEnumerable[] pipelineResults = await Task.WhenAll( + projects.Select(x => devOpsService.GetPipelinesAsync(x.Trim()))); // flatten arrays of pipelines by project into an array of pipelines - var pipelines = pipelineResults + BuildDefinition[] pipelines = pipelineResults .SelectMany(x => x) .ToArray(); - var repositoryUrls = GetDistinctRepositoryUrls(pipelines); + string[] repositoryUrls = GetDistinctRepositoryUrls(pipelines); - var codeOwnerEntriesByRepository = await GetCodeOwnerEntriesAsync(repositoryUrls); + Dictionary> codeownersEntriesByRepository = + await GetCodeownersEntriesAsync(repositoryUrls); - var pipelineOwners = await AssociateOwnersToPipelinesAsync(pipelines, codeOwnerEntriesByRepository); + List<(BuildDefinition Pipeline, List Owners)> pipelineOwners = + await AssociateOwnersToPipelinesAsync(pipelines, codeownersEntriesByRepository); - var outputContent = pipelineOwners.ToDictionary(x => x.Pipeline.Id, x => x.Owners); + Dictionary> outputContent = + pipelineOwners.ToDictionary(x => x.Pipeline.Id, x => x.Owners); - await File.WriteAllTextAsync(this.settings.Output, JsonConvert.SerializeObject(outputContent, Formatting.Indented), stoppingToken); + await File.WriteAllTextAsync( + this.settings.Output, + JsonConvert.SerializeObject(outputContent, Formatting.Indented), + stoppingToken); } private async Task Owners)>> AssociateOwnersToPipelinesAsync( IEnumerable pipelines, - Dictionary> codeOwnerEntriesByRepository) + Dictionary> codeownersEntriesByRepository) { - var linkedGithubUsers = await githubToAadResolver.GetPeopleLinksAsync(); + UserLink[] linkedGithubUsers = await githubToAadResolver.GetPeopleLinksAsync(); - var microsoftAliasMap = linkedGithubUsers.ToDictionary(x => x.GitHub.Login, x => x.Aad.UserPrincipalName, StringComparer.OrdinalIgnoreCase); + Dictionary microsoftAliasMap = linkedGithubUsers.ToDictionary( + x => x.GitHub.Login, + x => x.Aad.UserPrincipalName, + StringComparer.OrdinalIgnoreCase); - var microsoftPipelineOwners = new List<(BuildDefinition Pipeline, List Owners)>(); + List<(BuildDefinition Pipeline, List Owners)> microsoftPipelineOwners = + new List<(BuildDefinition Pipeline, List Owners)>(); - var unrecognizedGitHubAliases = new HashSet(); + HashSet unrecognizedGitHubAliases = new HashSet(); - foreach (var pipeline in pipelines) + foreach (BuildDefinition pipeline in pipelines) { if (pipeline.Process.Type != PipelineYamlProcessType || !(pipeline.Process is YamlProcess process)) { @@ -87,32 +98,44 @@ public async Task ExecuteAsync(CancellationToken stoppingToken = default) if (pipeline.Repository.Type != "GitHub") { - logger.LogInformation("Skipping pipeline '{Pipeline}' with {Type} repository", pipeline.Name, pipeline.Repository.Type); + logger.LogInformation( + "Skipping pipeline '{Pipeline}' with {Type} repository", + pipeline.Name, + pipeline.Repository.Type); continue; } - if (!codeOwnerEntriesByRepository.TryGetValue(SanitizeRepositoryUrl(pipeline.Repository.Url.AbsoluteUri), out var codeOwnerEntries)) + if (!codeownersEntriesByRepository.TryGetValue( + SanitizeRepositoryUrl(pipeline.Repository.Url.AbsoluteUri), + out List codeownersEntries)) { - logger.LogInformation("Skipping pipeline '{Pipeline}' because its repo has no CODEOWNERS file", pipeline.Name); + logger.LogInformation( + "Skipping pipeline '{Pipeline}' because its repo has no CODEOWNERS file", + pipeline.Name); continue; } logger.LogInformation("Processing pipeline '{Pipeline}'", pipeline.Name); - logger.LogInformation("Searching CODEOWNERS for patch matching {Path}", process.YamlFilename); - var codeOwnerEntry = CodeOwnersFile.FindOwnersForClosestMatch(codeOwnerEntries, process.YamlFilename); - codeOwnerEntry.FilterOutNonUserAliases(); + string buildDefPath = process.YamlFilename; + logger.LogInformation("Searching CODEOWNERS for patch matching {Path}", buildDefPath); + CodeownersEntry codeownersEntry = + CodeownersFile.GetMatchingCodeownersEntry(buildDefPath, codeownersEntries); + codeownersEntry.ExcludeNonUserAliases(); - logger.LogInformation("Matching Path = {Path}, Owner Count = {OwnerCount}", process.YamlFilename, codeOwnerEntry.Owners.Count); + logger.LogInformation( + "Matching Path = {Path}, Owner Count = {OwnerCount}", + buildDefPath, + codeownersEntry.Owners.Count); // Get set of team members in the CODEOWNERS file - var githubOwners = codeOwnerEntry.Owners.ToArray(); + string[] githubOwners = codeownersEntry.Owners.ToArray(); - var microsoftOwners = new List(); + List microsoftOwners = new List(); - foreach (var githubOwner in githubOwners) + foreach (string githubOwner in githubOwners) { - if (microsoftAliasMap.TryGetValue(githubOwner, out var microsoftOwner)) + if (microsoftAliasMap.TryGetValue(githubOwner, out string microsoftOwner)) { microsoftOwners.Add(microsoftOwner); } @@ -125,25 +148,28 @@ public async Task ExecuteAsync(CancellationToken stoppingToken = default) microsoftPipelineOwners.Add((pipeline, microsoftOwners)); } - var mappedCount = microsoftPipelineOwners.SelectMany(x => x.Owners).Distinct().Count(); - logger.LogInformation("{Mapped} unique pipeline owner aliases mapped to Microsoft users. {Unmapped} could not be mapped.", mappedCount, unrecognizedGitHubAliases.Count); + int mappedCount = microsoftPipelineOwners.SelectMany(x => x.Owners).Distinct().Count(); + logger.LogInformation( + "{Mapped} unique pipeline owner aliases mapped to Microsoft users. {Unmapped} could not be mapped.", + mappedCount, + unrecognizedGitHubAliases.Count); return microsoftPipelineOwners; } - private async Task>> GetCodeOwnerEntriesAsync(string[] repositoryUrls) + private async Task>> GetCodeownersEntriesAsync(string[] repositoryUrls) { - var tasks = repositoryUrls + IEnumerable Codeowners)>> tasks = repositoryUrls .Select(SanitizeRepositoryUrl) .Select(async url => ( RepositoryUrl: url, - CodeOwners: await this.gitHubService.GetCodeownersFile(new Uri(url)) + Codeowners: await this.gitHubService.GetCodeownersFileEntries(new Uri(url)) )); - var taskResults = await Task.WhenAll(tasks); + (string RepositoryUrl, List Codeowners)[] taskResults = await Task.WhenAll(tasks); - return taskResults.Where(x => x.CodeOwners != null) - .ToDictionary(x => x.RepositoryUrl, x => x.CodeOwners, StringComparer.OrdinalIgnoreCase); + return taskResults.Where(x => x.Codeowners != null) + .ToDictionary(x => x.RepositoryUrl, x => x.Codeowners, StringComparer.OrdinalIgnoreCase); } private static string SanitizeRepositoryUrl(string url) => Regex.Replace(url, @"\.git$", string.Empty); diff --git a/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Program.cs b/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Program.cs index ea781839dbd..cfd0199db6f 100644 --- a/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Program.cs +++ b/tools/pipeline-owners-extractor/Azure.Sdk.Tools.PipelineOwnersExtractor/Program.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.IO; using System.Reflection; using System.Threading.Tasks; @@ -31,7 +31,9 @@ public static async Task Main(string[] args) services.AddSingleton(); services.AddSingleton(); services.Configure(context.Configuration); - services.AddSingleton, PostConfigureKeyVaultSettings>(); + services + .AddSingleton, + PostConfigureKeyVaultSettings>(); services.AddSingleton(); services.AddSingleton(CreateGithubAadConverter); services.AddSingleton(CreateAzureDevOpsService); diff --git a/tools/pipeline-owners-extractor/PipelineOwnersExtractor.sln.DotSettings b/tools/pipeline-owners-extractor/PipelineOwnersExtractor.sln.DotSettings new file mode 100644 index 00000000000..0c17e9fb7b2 --- /dev/null +++ b/tools/pipeline-owners-extractor/PipelineOwnersExtractor.sln.DotSettings @@ -0,0 +1,2 @@ + + True \ No newline at end of file