Skip to content

Commit

Permalink
CodeOwnersParser Team/User lookup (#6366)
Browse files Browse the repository at this point in the history
* Codeowners Team/User lookup

* Add a TeamUserHolder and pass that to Owners parsing. Remove the manual tester project

* Add command line option to RetrieveCodeOwners to output json to a file

* Add team/storage URI to the deserialize error message in the TeamUserHolder
  • Loading branch information
JimSuplizio authored Jun 29, 2023
1 parent 2964ccf commit 5ad8b9e
Show file tree
Hide file tree
Showing 5 changed files with 177 additions and 26 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ public static class Program
/// Defaults to ".git".
/// Example usage: ".git|foo|bar"
/// </param>
/// <param name="teamStorageURI">Override for the default URI where the team/storage blob data resides.</param>
/// <param name="ownersDataOutputFile">File to output the owners data to, will overwrite if the file exist.</param>
/// <returns>
/// On STDOUT: The JSON representation of the matched CodeownersEntry.
/// "new CodeownersEntry()" if no path in the CODEOWNERS data matches.
Expand All @@ -48,7 +50,9 @@ public static int Main(
string codeownersFilePathOrUrl,
bool excludeNonUserAliases = false,
string? targetDir = null,
string ignoredPathPrefixes = DefaultIgnoredPrefixes)
string ignoredPathPrefixes = DefaultIgnoredPrefixes,
string? teamStorageURI = null,
string? ownersDataOutputFile = null)
{
try
{
Expand All @@ -71,17 +75,29 @@ public static int Main(
targetDir!,
codeownersFilePathOrUrl,
excludeNonUserAliases,
SplitIgnoredPathPrefixes())
SplitIgnoredPathPrefixes(),
teamStorageURI)
: GetCodeownersForSimplePath(
targetPath,
codeownersFilePathOrUrl,
excludeNonUserAliases);
excludeNonUserAliases,
teamStorageURI);

string codeownersJson = JsonSerializer.Serialize(
codeownersData,
new JsonSerializerOptions { WriteIndented = true });

Console.WriteLine(codeownersJson);

// If the output data file is specified, write the json to that.
if (!string.IsNullOrEmpty(ownersDataOutputFile))
{
// False in the ctor is to overwrite, not append
using (StreamWriter outputFile = new StreamWriter(ownersDataOutputFile, false))
{
outputFile.WriteLine(codeownersJson);
}
}
return 0;

string[] SplitIgnoredPathPrefixes()
Expand All @@ -101,7 +117,8 @@ private static Dictionary<string, CodeownersEntry> GetCodeownersForGlobPath(
string targetDir,
string codeownersFilePathOrUrl,
bool excludeNonUserAliases,
string[]? ignoredPathPrefixes = null)
string[]? ignoredPathPrefixes = null,
string? teamStorageURI=null)
{
ignoredPathPrefixes ??= Array.Empty<string>();

Expand All @@ -110,7 +127,8 @@ private static Dictionary<string, CodeownersEntry> GetCodeownersForGlobPath(
targetPath,
targetDir,
codeownersFilePathOrUrl,
ignoredPathPrefixes);
ignoredPathPrefixes,
teamStorageURI);

if (excludeNonUserAliases)
codeownersEntries.Values.ToList().ForEach(entry => entry.ExcludeNonUserAliases());
Expand All @@ -121,12 +139,14 @@ private static Dictionary<string, CodeownersEntry> GetCodeownersForGlobPath(
private static CodeownersEntry GetCodeownersForSimplePath(
string targetPath,
string codeownersFilePathOrUrl,
bool excludeNonUserAliases)
bool excludeNonUserAliases,
string? teamStorageURI = null)
{
CodeownersEntry codeownersEntry =
CodeownersFile.GetMatchingCodeownersEntry(
targetPath,
codeownersFilePathOrUrl);
codeownersFilePathOrUrl,
teamStorageURI);

if (excludeNonUserAliases)
codeownersEntry.ExcludeNonUserAliases();
Expand Down
50 changes: 41 additions & 9 deletions tools/code-owners-parser/CodeOwnersParser/CodeownersEntry.cs
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Serialization;

namespace Azure.Sdk.Tools.CodeOwnersParser
{
Expand Down Expand Up @@ -45,7 +48,7 @@ public CodeownersEntry(string pathExpression, List<string> owners)
}

private static string[] SplitLine(string line, char splitOn)
=> line.Split(new char[] { splitOn }, StringSplitOptions.RemoveEmptyEntries);
=> line.Split(new char[] { splitOn }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);

public override string ToString()
=> $"HasWildcard:{ContainsWildcard} Expression:{PathExpression} " +
Expand Down Expand Up @@ -85,14 +88,11 @@ private static IEnumerable<string> ParseLabels(string line, string moniker)
line = line[(colonPosition + 1)..].Trim();
foreach (string label in SplitLine(line, LabelSeparator).ToList())
{
if (!string.IsNullOrWhiteSpace(label))
{
yield return label.Trim();
}
yield return label;
}
}

public void ParseOwnersAndPath(string line)
public void ParseOwnersAndPath(string line, TeamUserHolder teamUserHolder)
{
if (
string.IsNullOrEmpty(line)
Expand All @@ -106,10 +106,42 @@ public void ParseOwnersAndPath(string line)
line = ParsePath(line);
line = RemoveCommentIfAny(line);

foreach (string author in SplitLine(line, OwnerSeparator).ToList())
// If the line doesn't contain the OwnerSeparator AKA no owners, then the foreach loop below
// won't work. For example, the following line would end up causing "/sdk/communication" to
// be added as an owner when one is not listed
// /sdk/communication/
if (line.Contains(OwnerSeparator))
{
foreach (string author in SplitLine(line, OwnerSeparator).ToList())
{
// If the author is a team, get the user list and add that to the Owners
if (!IsGitHubUserAlias(author))
{
var teamUsers = teamUserHolder.GetUsersForTeam(author);
// If the team is found in team user data, add the list of users to
// the owners and ensure the end result is a distinct list
if (teamUsers.Count > 0)
{
// The union of the two lists will ensure the result a distinct list
Owners = Owners.Union(teamUsers).ToList();
}
// Else, the team user data did not contain an entry or there were no user
// for the team. In that case, just add the team to the list of authors
else
{
Owners.Add(author);
}
}
// If the entry isn't a team, then just add it
else
{
Owners.Add(author);
}
}
}
else
{
if (!string.IsNullOrWhiteSpace(author))
Owners.Add(author.Trim());
Console.WriteLine($"Warning: CODEOWNERS line '{line}' does not have an owner entry.");
}
}

Expand Down
27 changes: 17 additions & 10 deletions tools/code-owners-parser/CodeOwnersParser/CodeownersFile.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,24 @@
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text.Json;

namespace Azure.Sdk.Tools.CodeOwnersParser
{
public static class CodeownersFile
{

public static List<CodeownersEntry> GetCodeownersEntriesFromFileOrUrl(
string codeownersFilePathOrUrl)
string codeownersFilePathOrUrl,
string? teamStorageURI = null)
{
string content = FileHelpers.GetFileOrUrlContents(codeownersFilePathOrUrl);
return GetCodeownersEntries(content);
return GetCodeownersEntries(content, teamStorageURI);
}

public static List<CodeownersEntry> GetCodeownersEntries(string codeownersContent)
public static List<CodeownersEntry> GetCodeownersEntries(string codeownersContent, string? teamStorageURI = null)
{
TeamUserHolder teamUserHolder = new TeamUserHolder(teamStorageURI);
List<CodeownersEntry> entries = new List<CodeownersEntry>();

// We are going to read line by line until we find a line that is not a comment
Expand All @@ -28,29 +32,31 @@ public static List<CodeownersEntry> GetCodeownersEntries(string codeownersConten
using StringReader sr = new StringReader(codeownersContent);
while (sr.ReadLine() is { } line)
{
entry = ProcessCodeownersLine(line, entry, entries);
entry = ProcessCodeownersLine(line, entry, entries, teamUserHolder);
}

return entries;
}

public static CodeownersEntry GetMatchingCodeownersEntry(
string targetPath,
string codeownersFilePathOrUrl)
string codeownersFilePathOrUrl,
string? teamStorageURI = null)
{
var codeownersEntries = GetCodeownersEntriesFromFileOrUrl(codeownersFilePathOrUrl);
var codeownersEntries = GetCodeownersEntriesFromFileOrUrl(codeownersFilePathOrUrl, teamStorageURI);
return GetMatchingCodeownersEntry(targetPath, codeownersEntries);
}

public static Dictionary<string, CodeownersEntry> GetMatchingCodeownersEntries(
GlobFilePath targetPath,
string targetDir,
string codeownersFilePathOrUrl,
string[]? ignoredPathPrefixes = null)
string[]? ignoredPathPrefixes = null,
string? teamStorageURI = null)
{
ignoredPathPrefixes ??= Array.Empty<string>();

var codeownersEntries = GetCodeownersEntriesFromFileOrUrl(codeownersFilePathOrUrl);
var codeownersEntries = GetCodeownersEntriesFromFileOrUrl(codeownersFilePathOrUrl, teamStorageURI);

Dictionary<string, CodeownersEntry> codeownersEntriesByPath = targetPath
.ResolveGlob(targetDir, ignoredPathPrefixes)
Expand All @@ -74,7 +80,8 @@ public static CodeownersEntry GetMatchingCodeownersEntry(
private static CodeownersEntry ProcessCodeownersLine(
string line,
CodeownersEntry entry,
List<CodeownersEntry> entries)
List<CodeownersEntry> entries,
TeamUserHolder teamUserHolder)
{
line = NormalizeLine(line);

Expand All @@ -85,7 +92,7 @@ private static CodeownersEntry ProcessCodeownersLine(

if (!IsCommentLine(line) || (IsCommentLine(line) && IsPlaceholderEntry(line)))
{
entry.ParseOwnersAndPath(line);
entry.ParseOwnersAndPath(line, teamUserHolder);

if (entry.IsValid)
entries.Add(entry);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace Azure.Sdk.Tools.CodeOwnersParser
{
public class DefaultStorageConstants
{
public const string DefaultStorageURI = "https://azuresdkartifacts.blob.core.windows.net/azure-sdk-write-teams/azure-sdk-write-teams-blob";
}
}
79 changes: 79 additions & 0 deletions tools/code-owners-parser/CodeOwnersParser/TeamUserHolder.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading.Tasks;

namespace Azure.Sdk.Tools.CodeOwnersParser
{
public class TeamUserHolder
{
private string TeamUserStorageURI { get; set; } = DefaultStorageConstants.DefaultStorageURI;
private Dictionary<string, List<string>>? _teamUserDict = null;

public Dictionary<string, List<string>> TeamUserDict
{
get
{
if (_teamUserDict == null)
{
_teamUserDict = GetTeamUserData();
}
return _teamUserDict;
}
set
{
_teamUserDict = value;
}
}

public TeamUserHolder(string? teamUserStorageURI)
{
if (!string.IsNullOrWhiteSpace(teamUserStorageURI))
{
TeamUserStorageURI = teamUserStorageURI;
}
}

private Dictionary<string, List<string>> GetTeamUserData()
{
if (null == _teamUserDict)
{
string rawJson = FileHelpers.GetFileOrUrlContents(TeamUserStorageURI);
var list = JsonSerializer.Deserialize<List<KeyValuePair<string, List<string>>>>(rawJson);
if (null != list)
{
return list.ToDictionary((keyItem) => keyItem.Key, (valueItem) => valueItem.Value);
}
Console.WriteLine($"Error! Unable to deserialize json team/user data from {TeamUserStorageURI}. rawJson={rawJson}");
return new Dictionary<string, List<string>>();
}
return _teamUserDict;
}

public List<string> GetUsersForTeam(string teamName)
{
// The teamName in the codeowners file should be in the form <org>/<team>.
// The dictionary's team names do not contain the org so the org needs to
// be stripped off. Handle the case where the teamName passed in does and
// does not being with @org/
string teamWithoutOrg = teamName.Trim();
if (teamWithoutOrg.Contains('/'))
{
teamWithoutOrg = teamWithoutOrg.Split("/")[1];
}
if (TeamUserDict != null)
{
if (TeamUserDict.ContainsKey(teamWithoutOrg))
{
Console.WriteLine($"Found team entry for {teamWithoutOrg}");
return TeamUserDict[teamWithoutOrg];
}
Console.WriteLine($"Warning: TeamUserDictionary did not contain a team entry for {teamWithoutOrg}");
}
return new List<string>();
}
}
}

0 comments on commit 5ad8b9e

Please sign in to comment.