Add GitHub Team User Store to tools sources (#6355)

* Add GitHub Team User Store to tools sources

* Output a warning if a team is found with no members

* Update to add command line parsing for storage and other feedback

* Remove unnecessary folder that was added and not used

* Pass in storage URI on command line, use BlobUriBuilder to parse the pieces

* Create the BlobUriBuilder as part of the GitHubEventClient and store that instead of the URI string

tools/github-team-user-store/GitHubTeamUserStore/GitHubTeamUserStore.sln

@@ -0,0 +1,25 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.5.33530.505
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GitHubTeamUserStore", "GitHubTeamUserStore\GitHubTeamUserStore.csproj", "{47699B24-3A45-47FC-B6ED-40717A3B568B}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{47699B24-3A45-47FC-B6ED-40717A3B568B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{47699B24-3A45-47FC-B6ED-40717A3B568B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{47699B24-3A45-47FC-B6ED-40717A3B568B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{47699B24-3A45-47FC-B6ED-40717A3B568B}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {54B86434-B8CB-48F9-99EA-3013047BC952}
+	EndGlobalSection
+EndGlobal

tools/github-team-user-store/GitHubTeamUserStore/GitHubTeamUserStore/Constants/ProductAndTeamConstants.cs

@@ -0,0 +1,17 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace GitHubTeamUserStore.Constants
+{
+    internal class ProductAndTeamConstants
+    {
+        // The ProductHeaderName is used to register the GitHubClient for this application
+        public const string ProductHeaderName = "azure-sdk-github-team-user-store";
+        // Need to do this since Octokit doesn't expose the API to get team by name.
+        // The team Id won't change even if the team name gets modified.
+        public const int AzureSdkWriteTeamId = 3057675;
+    }
+}

tools/github-team-user-store/GitHubTeamUserStore/GitHubTeamUserStore/GitHubEventClient.cs

@@ -0,0 +1,194 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Azure.Storage.Blobs;
+using Octokit;
+using GitHubTeamUserStore.Constants;
+
+namespace GitHubTeamUserStore
+{
+    public class GitHubEventClient
+    {
+        private const int MaxPageSize = 100;
+        // This needs to be done set because both GetAllMembers and GetAllChildTeams API calls auto-paginate
+        // but default to a page size of 30. Default to 100/page to reduce the number of API calls
+        private static ApiOptions _apiOptions = new ApiOptions() { PageSize = MaxPageSize };
+        public GitHubClient _gitHubClient = null;
+        public int CoreRateLimit { get; set; } = 0;
+        private int _numRetries = 5;
+        private int _delayTimeInMs = 1000;
+
+        private BlobUriBuilder AzureBlobUriBuilder { get; set; } = null;
+        public GitHubEventClient(string productHeaderName, string azureBlobStorageURI)
+        {
+            _gitHubClient = CreateClientWithGitHubEnvToken(productHeaderName);
+            Uri blobStorageUri = new Uri(azureBlobStorageURI);
+            BlobUriBuilder blobUriBuilder = new BlobUriBuilder(blobStorageUri);
+            AzureBlobUriBuilder = blobUriBuilder;
+        }
+
+        /// <summary>
+        /// This method creates a GitHubClient using the GITHUB_TOKEN from the environment for authentication
+        /// </summary>
+        /// <param name="productHeaderName">This is used to generate the User Agent string sent with each request. The name used should represent the product, the GitHub Organization, or the GitHub username that's using Octokit.net (in that order of preference).</param>
+        /// <exception cref="ArgumentException">If the product header name is null or empty</exception>
+        /// <exception cref="ApplicationException">If there is no GITHUB_TOKEN in the environment</exception>
+        /// <returns>Authenticated GitHubClient</returns>
+        public virtual GitHubClient CreateClientWithGitHubEnvToken(string productHeaderName)
+        {
+            if (string.IsNullOrEmpty(productHeaderName))
+            {
+                throw new ArgumentException("productHeaderName cannot be null or empty");
+            }
+            var githubToken = Environment.GetEnvironmentVariable("GITHUB_TOKEN");
+            if (string.IsNullOrEmpty(githubToken))
+            {
+                throw new ApplicationException("GITHUB_TOKEN cannot be null or empty");
+            }
+            var gitHubClient = new GitHubClient(new ProductHeaderValue(productHeaderName))
+            {
+                Credentials = new Credentials(githubToken)
+            };
+            return gitHubClient;
+        }
+
+        /// <summary>
+        /// Using the authenticated GitHubClient, call the RateLimit API to get the rate limits.
+        /// </summary>
+        /// <returns>Octokit.MiscellaneousRateLimit which contains the rate limit information.</returns>
+        public async Task<MiscellaneousRateLimit> GetRateLimits()
+        {
+            return await _gitHubClient.RateLimit.GetRateLimits();
+        }
+
+        /// <summary>
+        /// Write the current rate limit and remaining number of transactions.
+        /// </summary>
+        /// <param name="prependMessage">Optional message to prepend to the rate limit message.</param>
+        public async Task WriteRateLimits(string prependMessage = null)
+        {
+            var miscRateLimit = await GetRateLimits();
+            CoreRateLimit = miscRateLimit.Resources.Core.Limit;
+            // Get the Minutes till reset.
+            TimeSpan span = miscRateLimit.Resources.Core.Reset.UtcDateTime.Subtract(DateTime.UtcNow);
+            // In the message, cast TotalMinutes to an int to get a whole number of minutes.
+            string rateLimitMessage = $"Limit={miscRateLimit.Resources.Core.Limit}, Remaining={miscRateLimit.Resources.Core.Remaining}, Limit Reset in {(int)span.TotalMinutes} minutes.";
+            if (prependMessage != null)
+            {
+                rateLimitMessage = $"{prependMessage} {rateLimitMessage}";
+            }
+            Console.WriteLine(rateLimitMessage);
+        }
+
+        // Given a teamId, get the Team from github. Chances are, this is only going to be used to get
+        // the first team
+        public async Task<Team> GetTeamById(int teamId)
+        {
+            return await _gitHubClient.Organization.Team.Get(teamId);
+        }
+
+        /// <summary>
+        /// Given an Octokit.Team, call to get the team members. Note: GitHub's GetTeamMembers API gets all of the Users
+        /// for the team which includes all the members of child teams.
+        /// </summary>
+        /// <param name="team">Octokit.Team, the team whose members to retrieve.</param>
+        /// <returns>IReadOnlyList of Octokit.Users</returns>
+        /// <exception cref="ApplicationException">Thrown if GetAllMembers fails after all retries have been exhausted.</exception>
+        public async Task<IReadOnlyList<User>> GetTeamMembers(Team team)
+        {
+            // For the cases where exceptions/retries fail and an empty ReadOnlyList needs to be returned
+            List<User> emptyUserList = new List<User>();
+
+            int tryNumber = 0;
+            while (tryNumber < _numRetries)
+            {
+                tryNumber++;
+                try
+                {
+                    return await _gitHubClient.Organization.Team.GetAllMembers(team.Id, _apiOptions);
+                }
+                // This is what gets thrown if we try and get a userList for certain special teams on GitHub. 
+                // None of these teams are used directly in anything and neither team is a child team of 
+                // azure-sdk-write. If a ForbiddenException is encountered, then report it and return an
+                // empty list.
+                catch (Octokit.ForbiddenException forbiddenEx)
+                {
+                    Console.WriteLine($"{team.Name} cannot be retrieved using a GitHub PAT.");
+                    Console.WriteLine(forbiddenEx.Message);
+                    return emptyUserList.AsReadOnly();
+                }
+                // The only time we should get down here is if there's an exception caused by a network hiccup.
+                // Sleep for a second and try again.
+                catch (Exception ex)
+                {
+                    Console.WriteLine(ex.Message);
+                    Console.WriteLine($"delaying {_delayTimeInMs} and retrying");
+                    await Task.Delay(_delayTimeInMs);
+                }
+            }
+            throw new ApplicationException($"Unable to get members for team {team.Name}. See above exception(s)");
+        }
+
+        /// <summary>
+        /// Given an Octokit.Team, call to get all child teams.
+        /// </summary>
+        /// <param name="team">Octokit.Team, the team whose child teams to retrieve.</param>
+        /// <returns>IReadOnlyList of Octkit.Team</returns>
+        /// <exception cref="ApplicationException">Thrown if GetAllChildTeams fails after all retries have been exhausted</exception>
+        public async Task<IReadOnlyList<Team>> GetAllChildTeams(Team team)
+        {
+            int tryNumber = 0;
+            while (tryNumber < _numRetries)
+            {
+                tryNumber++;
+                try
+                {
+                    return await _gitHubClient.Organization.Team.GetAllChildTeams(team.Id, _apiOptions);
+                }
+                // The only time we should get down here is if there's an exception caused by a network hiccup.
+                // Sleep for a second and try again.
+                catch (Exception ex)
+                {
+                    Console.WriteLine(ex.Message);
+                    Console.WriteLine($"delaying {_delayTimeInMs} and retrying");
+                    await Task.Delay(_delayTimeInMs);
+                }
+            }
+            throw new ApplicationException($"Unable to get members for team {team.Name}. See above exception(s)");
+        }
+
+        /// <summary>
+        /// Store the team/user blob in Azure blob storage.
+        /// </summary>
+        /// <param name="rawJson">The json string, representing the team/user information, that will be uploaded to blob storage.</param>
+        /// <returns></returns>
+        /// <exception cref="ApplicationException">If there is no AZURE_SDK_TEAM_USER_STORE_SAS in the environment</exception>
+        public async Task UploadToBlobStorage(string rawJson)
+        {
+            BlobServiceClient blobServiceClient = new BlobServiceClient(AzureBlobUriBuilder.ToUri());
+            BlobContainerClient blobContainerClient = blobServiceClient.GetBlobContainerClient(AzureBlobUriBuilder.BlobContainerName);
+            BlobClient blobClient = blobContainerClient.GetBlobClient(AzureBlobUriBuilder.BlobName);
+            await blobClient.UploadAsync(BinaryData.FromString(rawJson), overwrite: true);
+        }
+
+        /// <summary>
+        /// Fetch the team/user blob date from Azure Blob storage.
+        /// </summary>
+        /// <returns>The raw json string blob.</returns>
+        /// <exception cref="ApplicationException">Thrown if the HttpResponseMessage does not contain a success status code.</exception>
+        public async Task<string> GetTeamUserBlobFromStorage()
+        {
+            HttpClient client = new HttpClient();
+            string blobUri = $"https://{AzureBlobUriBuilder.Host}/{AzureBlobUriBuilder.BlobContainerName}/{AzureBlobUriBuilder.BlobName}";
+            HttpResponseMessage response = await client.GetAsync(blobUri);
+            if (response.IsSuccessStatusCode)
+            {
+                string rawJson = await response.Content.ReadAsStringAsync();
+                return rawJson;
+            }
+            throw new ApplicationException($"Unable to retrieve team user data from blob storage. Status code: {response.StatusCode}, Reason {response.ReasonPhrase}");
+        }
+    }
+}

tools/github-team-user-store/GitHubTeamUserStore/GitHubTeamUserStore/GitHubTeamUserStore.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <ToolCommandName>github-team-user-store</ToolCommandName>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Azure.Storage.Blobs" Version="12.16.0" />
+    <PackageReference Include="Octokit" Version="5.0.2" />
+    <PackageReference Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
+  </ItemGroup>
+
+</Project>

tools/github-team-user-store/GitHubTeamUserStore/GitHubTeamUserStore/Program.cs

@@ -0,0 +1,62 @@
+using System.CommandLine;
+using System.Diagnostics;
+using GitHubTeamUserStore.Constants;
+
+namespace GitHubTeamUserStore
+{
+    internal class Program
+    {
+        static async Task Main(string[] args)
+        {
+            var blobStorageURIOption = new Option<string>
+                (name: "--blobStorageURI",
+                description: "The blob storage URI including the SAS.");
+            blobStorageURIOption.IsRequired = true;
+
+            var rootCommand = new RootCommand
+            {
+                blobStorageURIOption,
+            };
+            rootCommand.SetHandler(PopulateTeamUserData,
+                                   blobStorageURIOption);
+
+            int returnCode = await rootCommand.InvokeAsync(args);
+            Console.WriteLine($"Exiting with return code {returnCode}");
+            Environment.Exit(returnCode);
+        }
+
+        private static async Task<int> PopulateTeamUserData(string blobStorageURI)
+        {
+
+            // Default the returnCode code to non-zero. If everything is successful it'll be set to 0
+            int returnCode = 1;
+            Stopwatch stopWatch = new Stopwatch();
+            stopWatch.Start();
+
+            GitHubEventClient gitHubEventClient = new GitHubEventClient(ProductAndTeamConstants.ProductHeaderName, blobStorageURI);
+
+            await gitHubEventClient.WriteRateLimits("RateLimit at start of execution:");
+            await TeamUserGenerator.GenerateAndStoreTeamUserList(gitHubEventClient);
+            await gitHubEventClient.WriteRateLimits("RateLimit at end of execution:");
+            bool storedEqualsGenerated = await TeamUserGenerator.VerifyStoredTeamUsers(gitHubEventClient);
+
+            stopWatch.Stop();
+            TimeSpan ts = stopWatch.Elapsed;
+            string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
+                ts.Hours, ts.Minutes, ts.Seconds,
+                ts.Milliseconds / 10);
+            Console.WriteLine($"Total run time: {elapsedTime}");
+
+            if (storedEqualsGenerated)
+            {
+                Console.WriteLine("List stored successfully.");
+                returnCode = 0;
+            }
+            else
+            {
+                Console.WriteLine("There were issues with generated vs stored data. See above for specifics.");
+            }
+            return returnCode;
+        }
+    }
+}