From 74ff5851bc761baa9003bc2387de62107f4daf24 Mon Sep 17 00:00:00 2001 From: Gareth Healy Date: Mon, 8 Jan 2024 16:23:20 +0000 Subject: [PATCH] updated CreateWhoAreYouIssue to be abit cleaner --- .github/workflows/build.yaml | 6 +- .gitignore | 4 +- README.md | 4 +- scripts/run-redhat-cop.sh | 7 +- .../users/CreateWhoAreYouIssueCommand.java | 29 ++- .../githubstats/model/WhoAreYou.java | 29 +++ .../{MembersInfo.java => csv/Members.java} | 6 +- .../{RepoInfo.java => csv/Repository.java} | 36 ++-- .../githubstats/services/CsvService.java | 16 +- .../githubstats/services/GitHubService.java | 4 + .../services/stats/CollectStatsService.java | 12 +- ...ectRedHatLdapSupplementaryListService.java | 11 +- .../users/CreateWhoAreYouIssueService.java | 178 +++++++----------- .../GitHubMemberInRedHatLdapService.java | 26 +-- .../templates/CreateWhoAreYouIssue.ftl | 4 +- src/main/resources/reflection-config.json | 14 +- tests/supplementary.csv | 1 + 17 files changed, 201 insertions(+), 186 deletions(-) create mode 100644 src/main/java/com/garethahealy/githubstats/model/WhoAreYou.java rename src/main/java/com/garethahealy/githubstats/model/{MembersInfo.java => csv/Members.java} (86%) rename src/main/java/com/garethahealy/githubstats/model/{RepoInfo.java => csv/Repository.java} (79%) create mode 100644 tests/supplementary.csv diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index d5555ed..0158118 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -146,7 +146,7 @@ jobs: env: GITHUB_LOGIN: ${{ github.repository_owner }} GITHUB_OAUTH: ${{ secrets.RHUKI_READ_PAT }} - run: ./github-stats-*-runner create-who-are-you-issues --dry-run=true --organization=RedHat-Consulting-UK --issue-repo=helm3 --members-csv=tests/members.csv --fail-if-no-vpn=false + run: ./github-stats-*-runner create-who-are-you-issues --dry-run=true --organization=RedHat-Consulting-UK --issue-repo=helm3 --members-csv=tests/members.csv --supplementary-csv=tests/supplementary.csv --permission=write sign-image: needs: [ build ] @@ -170,7 +170,7 @@ jobs: cosign sign --yes ${image_uri} - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@91713af97dc80187565512baba96e4364e983601 # 0.16.0 + uses: aquasecurity/trivy-action@d43c1f16c00cfd3978dde6c07f4bbcf9eb6993ca # 0.16.1 env: TRIVY_USERNAME: ${{ github.repository_owner }} TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} @@ -181,7 +181,7 @@ jobs: output: "cosign-vuln.json" - name: Run Trivy SBOM generator - uses: aquasecurity/trivy-action@91713af97dc80187565512baba96e4364e983601 # 0.16.0 + uses: aquasecurity/trivy-action@d43c1f16c00cfd3978dde6c07f4bbcf9eb6993ca # 0.16.1 env: TRIVY_USERNAME: ${{ github.repository_owner }} TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 8fe9be3..2b0c5ba 100644 --- a/.gitignore +++ b/.gitignore @@ -39,5 +39,5 @@ nb-configuration.xml .env creds.source -gh-members.csv -supplementary.csv +/gh-members.csv +/supplementary.csv diff --git a/README.md b/README.md index 3e194e9..47b379d 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,8 @@ Loop over the GitHub members and see if we can find them in LDAP. Output what we ### GitHubMemberInRedHatLdap Loop over the GitHub members and see if we can find them in LDAP. Output what we find to a CSV. +`--supplementary-csv` is a list of known members that been created via `CollectRedHatLdapSupplementaryList` + ```bash ./target/github-stats-1.0.0-SNAPSHOT-runner github-member-in-ldap --dry-run=true --organization={your-org} --issue-repo={a-repo-in-that-org} --members-csv={list-of-known-members} --supplementary-csv={list-of-supplementary-members} -fail-if-no-vpn=false ``` @@ -62,5 +64,5 @@ Loop over the GitHub members and see if we can find them in LDAP. Output what we `--members-csv` is a list of known members that have validated their GitHub ID against their RH ID. See: `tests/members.csv` as an example. ```bash -./target/github-stats-1.0.0-SNAPSHOT-runner create-who-are-you-issues --dry-run=true --organization={your-org} --issue-repo={a-repo-in-that-org} --members-csv={list-of-known-members} --fail-if-no-vpn=false +./target/github-stats-1.0.0-SNAPSHOT-runner create-who-are-you-issues --dry-run=true --organization={your-org} --issue-repo={a-repo-in-that-org} --members-csv={list-of-known-members} --supplementary-csv={list-of-supplementary-members} --fail-if-no-vpn=false ``` \ No newline at end of file diff --git a/scripts/run-redhat-cop.sh b/scripts/run-redhat-cop.sh index 4f57cf1..0c78d58 100755 --- a/scripts/run-redhat-cop.sh +++ b/scripts/run-redhat-cop.sh @@ -4,7 +4,12 @@ scripts/download-memebers-sheet.sh ./mvnw clean install -Pnative +source creds.source + #./target/github-stats-1.0.0-SNAPSHOT-runner collect-stats --organization=redhat-cop ./target/github-stats-1.0.0-SNAPSHOT-runner collect-members-from-ldap --organization=redhat-cop --members-csv=gh-members.csv --csv-output=supplementary.csv --fail-if-no-vpn=true -./target/github-stats-1.0.0-SNAPSHOT-runner github-member-in-ldap --organization=redhat-cop --issue-repo=org --members-csv=gh-members.csv --supplementary-csv=supplementary.csv --fail-if-no-vpn=true \ No newline at end of file +./target/github-stats-1.0.0-SNAPSHOT-runner github-member-in-ldap --dry-run=true --organization=redhat-cop --issue-repo=org --members-csv=gh-members.csv --supplementary-csv=supplementary.csv --fail-if-no-vpn=true + +./target/github-stats-1.0.0-SNAPSHOT-runner create-who-are-you-issues --dry-run=true --organization=redhat-cop --issue-repo=org --members-csv=gh-members.csv --supplementary-csv=supplementary.csv --permission=admin +./target/github-stats-1.0.0-SNAPSHOT-runner create-who-are-you-issues --dry-run=true --organization=redhat-cop --issue-repo=org --members-csv=gh-members.csv --supplementary-csv=supplementary.csv --permission=write \ No newline at end of file diff --git a/src/main/java/com/garethahealy/githubstats/commands/users/CreateWhoAreYouIssueCommand.java b/src/main/java/com/garethahealy/githubstats/commands/users/CreateWhoAreYouIssueCommand.java index 61e5404..ae6730b 100644 --- a/src/main/java/com/garethahealy/githubstats/commands/users/CreateWhoAreYouIssueCommand.java +++ b/src/main/java/com/garethahealy/githubstats/commands/users/CreateWhoAreYouIssueCommand.java @@ -1,9 +1,11 @@ package com.garethahealy.githubstats.commands.users; import com.garethahealy.githubstats.services.users.CreateWhoAreYouIssueService; +import freemarker.template.TemplateException; import jakarta.enterprise.context.Dependent; import jakarta.inject.Inject; import org.apache.directory.api.ldap.model.exception.LdapException; +import org.kohsuke.github.GHPermissionType; import picocli.CommandLine; import java.io.IOException; @@ -24,8 +26,11 @@ public class CreateWhoAreYouIssueCommand implements Runnable { @CommandLine.Option(names = {"-i", "--members-csv"}, description = "CSV of current known members", required = true) String membersCsv; - @CommandLine.Option(names = {"-vpn", "--fail-if-no-vpn"}, description = "Throw an exception if can't connect to LDAP") - boolean failNoVpn; + @CommandLine.Option(names = {"-s", "--supplementary-csv"}, description = "CSV of current known members, generated by 'collect-members-from-ldap'", required = true) + String supplementaryCsv; + + @CommandLine.Option(names = {"-p", "--permission"}, description = "Permission to search against; ADMIN, WRITE, READ", required = true) + String permission; @Inject CreateWhoAreYouIssueService createWhoAreYouIssueService; @@ -35,9 +40,25 @@ public void run() { try { //TODO: for time being, always dry-run dryRun = true; - createWhoAreYouIssueService.run(organization, orgRepo, dryRun, membersCsv, failNoVpn); - } catch (IOException | LdapException e) { + + createWhoAreYouIssueService.run(organization, orgRepo, dryRun, membersCsv, supplementaryCsv, convert(permission)); + } catch (IOException | LdapException | TemplateException e) { throw new RuntimeException(e); } } + + private GHPermissionType convert(String permissions) { + GHPermissionType answer; + if (permissions.equalsIgnoreCase("ADMIN")) { + answer = GHPermissionType.ADMIN; + } else if (permissions.equalsIgnoreCase("WRITE")) { + answer = GHPermissionType.WRITE; + } else if (permissions.equalsIgnoreCase("READ")) { + answer = GHPermissionType.READ; + } else { + throw new IllegalArgumentException("--permission=" + permissions + " is invalid."); + } + + return answer; + } } \ No newline at end of file diff --git a/src/main/java/com/garethahealy/githubstats/model/WhoAreYou.java b/src/main/java/com/garethahealy/githubstats/model/WhoAreYou.java new file mode 100644 index 0000000..eb982bd --- /dev/null +++ b/src/main/java/com/garethahealy/githubstats/model/WhoAreYou.java @@ -0,0 +1,29 @@ +package com.garethahealy.githubstats.model; + +import io.quarkus.runtime.annotations.RegisterForReflection; +import org.apache.commons.lang3.builder.CompareToBuilder; + +@RegisterForReflection +public class WhoAreYou implements Comparable { + + private final String username; + private final String repo; + + public String getUsername() { + return username; + } + + public String getRepo() { + return repo; + } + + public WhoAreYou(String username, String repo) { + this.username = username; + this.repo = repo; + } + + @Override + public int compareTo(WhoAreYou o) { + return new CompareToBuilder().append(this.getUsername(), o.getUsername()).toComparison(); + } +} diff --git a/src/main/java/com/garethahealy/githubstats/model/MembersInfo.java b/src/main/java/com/garethahealy/githubstats/model/csv/Members.java similarity index 86% rename from src/main/java/com/garethahealy/githubstats/model/MembersInfo.java rename to src/main/java/com/garethahealy/githubstats/model/csv/Members.java index 99ba496..c3efeb8 100644 --- a/src/main/java/com/garethahealy/githubstats/model/MembersInfo.java +++ b/src/main/java/com/garethahealy/githubstats/model/csv/Members.java @@ -1,4 +1,4 @@ -package com.garethahealy.githubstats.model; +package com.garethahealy.githubstats.model.csv; import io.quarkus.runtime.annotations.RegisterForReflection; @@ -6,7 +6,7 @@ import java.util.List; @RegisterForReflection -public class MembersInfo { +public class Members { public enum Headers { Timestamp, EmailAddress, @@ -34,7 +34,7 @@ public String getWhatIsYourGitHubUsername() { return whatIsYourGitHubUsername; } - public MembersInfo(String timestamp, String emailAddress, String whatIsYourGitHubUsername) { + public Members(String timestamp, String emailAddress, String whatIsYourGitHubUsername) { this.timestamp = timestamp; this.emailAddress = emailAddress; this.whatIsYourGitHubUsername = whatIsYourGitHubUsername; diff --git a/src/main/java/com/garethahealy/githubstats/model/RepoInfo.java b/src/main/java/com/garethahealy/githubstats/model/csv/Repository.java similarity index 79% rename from src/main/java/com/garethahealy/githubstats/model/RepoInfo.java rename to src/main/java/com/garethahealy/githubstats/model/csv/Repository.java index 6322920..67d335a 100644 --- a/src/main/java/com/garethahealy/githubstats/model/RepoInfo.java +++ b/src/main/java/com/garethahealy/githubstats/model/csv/Repository.java @@ -1,4 +1,4 @@ -package com.garethahealy.githubstats.model; +package com.garethahealy.githubstats.model.csv; import org.kohsuke.github.*; @@ -9,7 +9,7 @@ import java.util.Arrays; import java.util.List; -public class RepoInfo { +public class Repository { public enum Headers { RepoName, @@ -51,22 +51,22 @@ public enum Headers { private final boolean inConfig; private final boolean isArchived; - public RepoInfo(String repoName, - GHCommit lastCommit, - List contributors, - List commits, - List issues, - List pullRequests, - List topics, - GHRepositoryCloneTraffic cloneTraffic, - GHRepositoryViewTraffic viewTraffic, - boolean hasOwners, - boolean hasCodeOwners, - boolean hasWorkflows, - boolean hasTravis, - boolean hasRenovate, - boolean inConfig, - boolean isArchived) throws IOException { + public Repository(String repoName, + GHCommit lastCommit, + List contributors, + List commits, + List issues, + List pullRequests, + List topics, + GHRepositoryCloneTraffic cloneTraffic, + GHRepositoryViewTraffic viewTraffic, + boolean hasOwners, + boolean hasCodeOwners, + boolean hasWorkflows, + boolean hasTravis, + boolean hasRenovate, + boolean inConfig, + boolean isArchived) throws IOException { DateFormat df = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss"); this.repoName = repoName; diff --git a/src/main/java/com/garethahealy/githubstats/services/CsvService.java b/src/main/java/com/garethahealy/githubstats/services/CsvService.java index dba589b..b3ce271 100644 --- a/src/main/java/com/garethahealy/githubstats/services/CsvService.java +++ b/src/main/java/com/garethahealy/githubstats/services/CsvService.java @@ -1,6 +1,6 @@ package com.garethahealy.githubstats.services; -import com.garethahealy.githubstats.model.MembersInfo; +import com.garethahealy.githubstats.model.csv.Members; import jakarta.enterprise.context.ApplicationScoped; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVRecord; @@ -14,21 +14,21 @@ @ApplicationScoped public class CsvService { - public Map getKnownMembers(String membersCsv) throws IOException { - Map answer = new HashMap<>(); + public Map getKnownMembers(String membersCsv) throws IOException { + Map answer = new HashMap<>(); CSVFormat csvFormat = CSVFormat.Builder.create(CSVFormat.DEFAULT) - .setHeader(MembersInfo.Headers.class) + .setHeader(Members.Headers.class) .setSkipHeaderRecord(true) .build(); try (Reader in = new FileReader(membersCsv)) { Iterable records = csvFormat.parse(in); for (CSVRecord record : records) { - String timestamp = record.get(MembersInfo.Headers.Timestamp); - String redhatEmail = record.get(MembersInfo.Headers.EmailAddress); - String username = record.get(MembersInfo.Headers.WhatIsYourGitHubUsername); + String timestamp = record.get(Members.Headers.Timestamp); + String redhatEmail = record.get(Members.Headers.EmailAddress); + String username = record.get(Members.Headers.WhatIsYourGitHubUsername); - answer.put(username, new MembersInfo(timestamp, redhatEmail, username)); + answer.put(username, new Members(timestamp, redhatEmail, username)); } } diff --git a/src/main/java/com/garethahealy/githubstats/services/GitHubService.java b/src/main/java/com/garethahealy/githubstats/services/GitHubService.java index 64434f4..04cd569 100644 --- a/src/main/java/com/garethahealy/githubstats/services/GitHubService.java +++ b/src/main/java/com/garethahealy/githubstats/services/GitHubService.java @@ -148,4 +148,8 @@ private boolean hasFileContent(GHRepository repo, String path) { return answer; } + + public PagedIterable listTeams(GHOrganization org) throws IOException { + return org.listTeams(); + } } diff --git a/src/main/java/com/garethahealy/githubstats/services/stats/CollectStatsService.java b/src/main/java/com/garethahealy/githubstats/services/stats/CollectStatsService.java index e409955..66ba71d 100644 --- a/src/main/java/com/garethahealy/githubstats/services/stats/CollectStatsService.java +++ b/src/main/java/com/garethahealy/githubstats/services/stats/CollectStatsService.java @@ -1,6 +1,6 @@ package com.garethahealy.githubstats.services.stats; -import com.garethahealy.githubstats.model.RepoInfo; +import com.garethahealy.githubstats.model.csv.Repository; import com.garethahealy.githubstats.services.GitHubService; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; @@ -47,13 +47,13 @@ public void run(String organization, boolean validateOrgConfig, String output) t logger.infof("Found %s repos.", repos.size()); CSVFormat csvFormat = CSVFormat.Builder.create(CSVFormat.DEFAULT) - .setHeader((RepoInfo.Headers.class)) + .setHeader((Repository.Headers.class)) .build(); int cores = Runtime.getRuntime().availableProcessors() * 2; try (CSVPrinter csvPrinter = new CSVPrinter(Files.newBufferedWriter(Paths.get(output)), csvFormat)) { try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { - List> futures = new ArrayList<>(); + List> futures = new ArrayList<>(); for (Map.Entry current : repos.entrySet()) { futures.add(executor.submit(() -> { logger.infof("Working on: %s", current.getValue().getName()); @@ -94,12 +94,12 @@ public void run(String organization, boolean validateOrgConfig, String output) t inConfig = configContent.contains(repoName); } - return new RepoInfo(repoName, lastCommit, contributors, commits, issues, pullRequests, topics, cloneTraffic, viewTraffic, + return new Repository(repoName, lastCommit, contributors, commits, issues, pullRequests, topics, cloneTraffic, viewTraffic, hasOwners, hasCodeOwners, hasWorkflows, hasTravis, hasRenovate, inConfig, isArchived); })); if (futures.size() == cores) { - for (Future future : futures) { + for (Future future : futures) { csvPrinter.printRecord(future.get().toArray()); } @@ -110,7 +110,7 @@ public void run(String organization, boolean validateOrgConfig, String output) t } } - for (Future future : futures) { + for (Future future : futures) { csvPrinter.printRecord(future.get().toArray()); } } diff --git a/src/main/java/com/garethahealy/githubstats/services/users/CollectRedHatLdapSupplementaryListService.java b/src/main/java/com/garethahealy/githubstats/services/users/CollectRedHatLdapSupplementaryListService.java index 079f591..5fae0d8 100644 --- a/src/main/java/com/garethahealy/githubstats/services/users/CollectRedHatLdapSupplementaryListService.java +++ b/src/main/java/com/garethahealy/githubstats/services/users/CollectRedHatLdapSupplementaryListService.java @@ -1,6 +1,6 @@ package com.garethahealy.githubstats.services.users; -import com.garethahealy.githubstats.model.MembersInfo; +import com.garethahealy.githubstats.model.csv.Members; import com.garethahealy.githubstats.services.CsvService; import com.garethahealy.githubstats.services.GitHubService; import com.garethahealy.githubstats.services.LdapService; @@ -42,20 +42,23 @@ public void run(String organization, String output, String membersCsv, boolean f GHOrganization org = gitHubService.getOrganization(gitHubService.getGitHub(), organization); List members = gitHubService.listMembers(org); - Map knownMembers = csvService.getKnownMembers(membersCsv); + Map knownMembers = csvService.getKnownMembers(membersCsv); CSVFormat csvFormat = CSVFormat.Builder.create(CSVFormat.DEFAULT) - .setHeader((MembersInfo.Headers.class)) + .setHeader((Members.Headers.class)) .build(); try (CSVPrinter csvPrinter = new CSVPrinter(Files.newBufferedWriter(Paths.get(output)), csvFormat)) { + // Hard code the bot user to be ignored + csvPrinter.printRecord(new Members("", "ablock@redhat.com", "redhat-cop-ci-bot").toArray()); + if (ldapService.canConnect()) { try (LdapConnection connection = ldapService.open()) { for (GHUser user : members) { if (!knownMembers.containsKey(user.getLogin())) { String email = ldapService.searchOnGitHub(connection, user.getLogin()); if (!email.isEmpty()) { - csvPrinter.printRecord(new MembersInfo("", email, user.getLogin()).toArray()); + csvPrinter.printRecord(new Members("", email, user.getLogin()).toArray()); } } } diff --git a/src/main/java/com/garethahealy/githubstats/services/users/CreateWhoAreYouIssueService.java b/src/main/java/com/garethahealy/githubstats/services/users/CreateWhoAreYouIssueService.java index 3da9c10..5258298 100644 --- a/src/main/java/com/garethahealy/githubstats/services/users/CreateWhoAreYouIssueService.java +++ b/src/main/java/com/garethahealy/githubstats/services/users/CreateWhoAreYouIssueService.java @@ -1,29 +1,21 @@ package com.garethahealy.githubstats.services.users; -import com.garethahealy.githubstats.model.MembersInfo; +import com.garethahealy.githubstats.model.WhoAreYou; +import com.garethahealy.githubstats.model.csv.Members; +import com.garethahealy.githubstats.services.CsvService; import com.garethahealy.githubstats.services.GitHubService; +import freemarker.template.Template; +import freemarker.template.TemplateException; +import io.quarkiverse.freemarker.TemplatePath; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVRecord; -import org.apache.directory.api.ldap.model.cursor.EntryCursor; -import org.apache.directory.api.ldap.model.entry.Entry; import org.apache.directory.api.ldap.model.exception.LdapException; -import org.apache.directory.api.ldap.model.message.SearchScope; -import org.apache.directory.api.ldap.model.name.Dn; -import org.apache.directory.ldap.client.api.LdapConnection; -import org.apache.directory.ldap.client.api.LdapNetworkConnection; -import org.apache.directory.ldap.client.api.exception.InvalidConnectionException; import org.jboss.logging.Logger; import org.kohsuke.github.*; -import java.io.FileReader; import java.io.IOException; -import java.io.Reader; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import java.io.StringWriter; +import java.util.*; @ApplicationScoped public class CreateWhoAreYouIssueService { @@ -31,130 +23,88 @@ public class CreateWhoAreYouIssueService { @Inject Logger logger; + @Inject + @TemplatePath("CreateWhoAreYouIssue.ftl") + Template issue; + private final GitHubService gitHubService; + private final CsvService csvService; @Inject - public CreateWhoAreYouIssueService(GitHubService gitHubService) { + public CreateWhoAreYouIssueService(GitHubService gitHubService, CsvService csvService) { this.gitHubService = gitHubService; + this.csvService = csvService; } - public void run(String organization, String issueRepo, boolean isDryRun, String membersCsv, boolean failNoVpn) throws IOException, LdapException { + public void run(String organization, String issueRepo, boolean isDryRun, String membersCsv, String supplementaryCsv, GHPermissionType perms) throws IOException, LdapException, TemplateException { GitHub gitHub = gitHubService.getGitHub(); - GHOrganization org = gitHub.getOrganization(organization); + GHOrganization org = gitHubService.getOrganization(gitHub, organization); - GHRepository orgRepo = org.getRepository(issueRepo); - List members = org.listMembers().toList(); + GHRepository orgRepo = gitHubService.getRepository(org, issueRepo); + List members = gitHubService.listMembers(org); - Set usernamesToIgnore = getUsernamesToIgnore(membersCsv); + Map knownMembers = csvService.getKnownMembers(membersCsv); + Map supplementaryMembers = csvService.getKnownMembers(supplementaryCsv); - logger.infof("There are %s members", members.size()); - logger.infof("There are %s members we already have emails for who will be ignored", usernamesToIgnore.size()); + logger.infof("There are %s GitHub members", members.size()); + logger.infof("There are %s known members and %s supplementary members in the CSVs", knownMembers.size(), supplementaryMembers.size()); - for (GHUser current : members) { - if (usernamesToIgnore.contains(current.getLogin())) { - logger.infof("Ignoring: %s", current.getLogin()); - } else { - GHIssueBuilder builder = orgRepo.createIssue("@" + current.getLogin() + " please complete form") - .assignee(current) - .label("admin") - .body("To be a member of the Red Hat CoP GitHub org, you are required to be a Red Hat employee. " + - "Non-employees are invited to be outside-collaborators (https://github.com/orgs/redhat-cop/outside-collaborators). " + - "As we currently do not know who is an employee and who is not, we are requiring all members to submit the following google form " + - "so that we can verify who are employees: " + - "https://red.ht/github-redhat-cop-username"); - - if (isDryRun) { - logger.infof("DRY-RUN: Would have created issue for %s in %s", current.getLogin(), orgRepo.getName()); - } else { - //TODO: check if issue already exists - builder.create(); - - logger.infof("Created issue for %s", current.getLogin()); - } - } - } + List usersToInform = collectUnknownUsers(org, knownMembers, supplementaryMembers, perms); + createIssue(usersToInform, orgRepo, perms, isDryRun); logger.info("Issues DONE"); - logger.infof("RateLimit: limit %s, remaining %s, resetDate %s", gitHub.getRateLimit().getLimit(), gitHub.getRateLimit().getRemaining(), gitHub.getRateLimit().getResetDate()); - - Set membersLogins = getMembersLogins(members); - for (String current : usernamesToIgnore) { - if (!membersLogins.contains(current)) { - logger.infof("Have a google form response but they are not part the github org anymore for %s", current); - } - } + } - logger.info("Responses to GH Org Lookup DONE"); - - //ldapsearch -x -h ldap.corp.redhat.com -b dc=redhat,dc=com -s sub 'uid=gahealy' - Dn systemDn = new Dn("dc=redhat,dc=com"); - try (LdapConnection connection = new LdapNetworkConnection("ldap.corp.redhat.com")) { - for (String current : getCollectedEmails(membersCsv)) { - String uid = current.split("@")[0]; - try (EntryCursor cursor = connection.search(systemDn, "(uid=" + uid + ")", SearchScope.SUBTREE)) { - boolean found = false; - for (Entry entry : cursor) { - entry.getDn(); - logger.infof("Found %s in ldap", uid); - found = true; - } + private List collectUnknownUsers(GHOrganization org, Map knownMembers, Map supplementaryMembers, GHPermissionType perms) throws IOException { + Map usersToInform = new HashMap<>(); + for (GHTeam team : gitHubService.listTeams(org)) { + logger.infof("Working on team %s", team.getName()); - if (!found) { - logger.infof("Did not find %s in ldap", uid); + for (GHUser member : team.getMembers()) { + if (knownMembers.containsKey(member.getLogin()) || supplementaryMembers.containsKey(member.getLogin()) || usersToInform.containsKey(member.getLogin())) { + logger.debugf("Ignoring: %s", member.getLogin()); + } else { + boolean foundUser = false; + for (GHRepository repository : team.listRepositories()) { + boolean hasPermission = repository.hasPermission(member, perms); + if (hasPermission) { + logger.warnf("Member %s has %s on %s - but we don't know who they are", member.getLogin(), perms, repository.getName()); + + usersToInform.put(member.getLogin(), new WhoAreYou(member.getLogin(), repository.getHtmlUrl().toString())); + break; + } } } } - } catch (InvalidConnectionException ex) { - if (failNoVpn) { - logger.error("Unable to search ldap for users. Are you on the VPN?", ex); - } else { - logger.warn("Unable to search ldap for users. Are you on the VPN?", ex); - } } - logger.info("Ldap Lookup DONE"); + List sortedList = new ArrayList<>(usersToInform.values()); + Collections.sort(sortedList); + return sortedList; } - private Set getUsernamesToIgnore(String membersCsv) throws IOException { - Set answer = new HashSet<>(); - CSVFormat csvFormat = CSVFormat.Builder.create(CSVFormat.DEFAULT) - .setHeader(MembersInfo.Headers.class) - .setSkipHeaderRecord(true) - .build(); - - try (Reader in = new FileReader(membersCsv)) { - Iterable records = csvFormat.parse(in); - for (CSVRecord record : records) { - answer.add(record.get(MembersInfo.Headers.WhatIsYourGitHubUsername)); - } - } + private void createIssue(List usersToInform, GHRepository orgRepo, GHPermissionType permissions, boolean isDryRun) throws TemplateException, IOException { + if (!usersToInform.isEmpty()) { + Map root = new HashMap<>(); + root.put("users", usersToInform); + root.put("permissions", permissions.toString()); - return answer; - } + StringWriter stringWriter = new StringWriter(); + issue.process(root, stringWriter); - private List getCollectedEmails(String membersCsv) throws IOException { - List answer = new ArrayList<>(); - CSVFormat csvFormat = CSVFormat.Builder.create(CSVFormat.DEFAULT) - .setHeader(MembersInfo.Headers.class) - .setSkipHeaderRecord(true) - .build(); - - try (Reader in = new FileReader(membersCsv)) { - Iterable records = csvFormat.parse(in); - for (CSVRecord record : records) { - answer.add(record.get(MembersInfo.Headers.EmailAddress)); - } - } - - return answer; - } + if (isDryRun) { + logger.warnf("DRY-RUN: Would have created issue in %s", orgRepo.getName()); + logger.warnf(stringWriter.toString()); + } else { + GHIssue createdIssue = orgRepo.createIssue("Request GitHub to Red Hat ID linkage for users with " + permissions) + .label("admin") + .body(stringWriter.toString()) + .create(); - private Set getMembersLogins(List members) { - Set answer = new HashSet<>(); - for (GHUser current : members) { - answer.add(current.getLogin()); + logger.infof("Created issue: %s", createdIssue.getUrl()); + } } - return answer; + logger.info("--> Issue creation DONE"); } } diff --git a/src/main/java/com/garethahealy/githubstats/services/users/GitHubMemberInRedHatLdapService.java b/src/main/java/com/garethahealy/githubstats/services/users/GitHubMemberInRedHatLdapService.java index ab2268c..6d144e4 100644 --- a/src/main/java/com/garethahealy/githubstats/services/users/GitHubMemberInRedHatLdapService.java +++ b/src/main/java/com/garethahealy/githubstats/services/users/GitHubMemberInRedHatLdapService.java @@ -1,6 +1,6 @@ package com.garethahealy.githubstats.services.users; -import com.garethahealy.githubstats.model.MembersInfo; +import com.garethahealy.githubstats.model.csv.Members; import com.garethahealy.githubstats.services.CsvService; import com.garethahealy.githubstats.services.GitHubService; import com.garethahealy.githubstats.services.LdapService; @@ -50,22 +50,22 @@ public void run(String organization, String issueRepo, boolean isDryRun, String GHRepository orgRepo = gitHubService.getRepository(org, issueRepo); List members = gitHubService.listMembers(org); - Map knownMembers = csvService.getKnownMembers(membersCsv); - Map supplementaryMembers = csvService.getKnownMembers(supplementaryCsv); + Map knownMembers = csvService.getKnownMembers(membersCsv); + Map supplementaryMembers = csvService.getKnownMembers(supplementaryCsv); logger.infof("There are %s GitHub members", members.size()); logger.infof("There are %s known members and %s supplementary members in the CSVs", knownMembers.size(), supplementaryMembers.size()); - List ldapCheck = collectLdapCheckList(members, knownMembers, supplementaryMembers); - List usersToRemove = searchFor(ldapCheck, failNoVpn); + List ldapCheck = collectLdapCheckList(members, knownMembers, supplementaryMembers); + List usersToRemove = searchFor(ldapCheck, failNoVpn); createIssue(usersToRemove, orgRepo, isDryRun); logger.info("Finished."); } - private List collectLdapCheckList(List members, Map knownMembers, Map supplementaryMembers) { - List answer = new ArrayList<>(); + private List collectLdapCheckList(List members, Map knownMembers, Map supplementaryMembers) { + List answer = new ArrayList<>(); for (GHUser current : members) { if (knownMembers.containsKey(current.getLogin())) { logger.infof("Adding %s to LDAP check list from known members", current.getLogin()); @@ -84,11 +84,11 @@ private List collectLdapCheckList(List members, Map searchFor(List ldapCheck, boolean failNoVpn) throws IOException, LdapException { - List answer = new ArrayList<>(); + private List searchFor(List ldapCheck, boolean failNoVpn) throws IOException, LdapException { + List answer = new ArrayList<>(); if (ldapService.canConnect()) { try (LdapConnection connection = ldapService.open()) { - for (MembersInfo current : ldapCheck) { + for (Members current : ldapCheck) { boolean found = ldapService.searchOnUser(connection, current.getRedHatUserId()); if (!found) { logger.warnf("Did not find %s in LDAP", current.getRedHatUserId()); @@ -106,7 +106,7 @@ private List searchFor(List ldapCheck, boolean failNoV return answer; } - private void createIssue(List usersToRemove, GHRepository orgRepo, boolean isDryRun) throws TemplateException, IOException { + private void createIssue(List usersToRemove, GHRepository orgRepo, boolean isDryRun) throws TemplateException, IOException { if (!usersToRemove.isEmpty()) { Map root = new HashMap<>(); root.put("users", usersToRemove); @@ -115,8 +115,8 @@ private void createIssue(List usersToRemove, GHRepository orgRepo, issue.process(root, stringWriter); if (isDryRun) { - logger.infof("DRY-RUN: Would have created issue in %s", orgRepo.getName()); - logger.infof(stringWriter.toString()); + logger.warnf("DRY-RUN: Would have created issue in %s", orgRepo.getName()); + logger.warnf(stringWriter.toString()); } else { GHIssue createdIssue = orgRepo.createIssue("Remove users - Not in RH LDAP") .label("admin") diff --git a/src/main/resources/freemarker/templates/CreateWhoAreYouIssue.ftl b/src/main/resources/freemarker/templates/CreateWhoAreYouIssue.ftl index 90f8b7f..0aea0b2 100644 --- a/src/main/resources/freemarker/templates/CreateWhoAreYouIssue.ftl +++ b/src/main/resources/freemarker/templates/CreateWhoAreYouIssue.ftl @@ -1,7 +1,7 @@ To be a member of the Red Hat CoP GitHub organization, you are required to be a Red Hat employee. Non-employees are invited to be outside-collaborators (https://github.com/orgs/redhat-cop/outside-collaborators). -To resolve GitHub IDs to Red Hat IDs, we check if a response of the below form has been provided, if not, we search LDAP. +To resolve GitHub IDs to Red Hat IDs, we check if a response to the below form has been provided, if not, we search LDAP. - https://red.ht/github-redhat-cop-username If you are unsure how to set your GitHub ID within LDAP, see: @@ -12,5 +12,5 @@ The below list of members have *${permissions}* and cannot be found using the ab Please complete the above form or add your GitHub handle to Rover. <#list users as user> -- @${user.getWhatIsYourGitHubUsername()} +- [ ] @${user.getUsername()} - has *${permissions}* on ${user.getRepo()} \ No newline at end of file diff --git a/src/main/resources/reflection-config.json b/src/main/resources/reflection-config.json index 70857c4..9d8359e 100644 --- a/src/main/resources/reflection-config.json +++ b/src/main/resources/reflection-config.json @@ -1,11 +1,11 @@ [ { - "name" : "com.github.benmanes.caffeine.cache.PSAMS", - "allDeclaredConstructors" : true, - "allPublicConstructors" : true, - "allDeclaredMethods" : true, - "allPublicMethods" : true, - "allDeclaredFields" : true, - "allPublicFields" : true + "name": "com.github.benmanes.caffeine.cache.PSAMS", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allDeclaredMethods": true, + "allPublicMethods": true, + "allDeclaredFields": true, + "allPublicFields": true } ] \ No newline at end of file diff --git a/tests/supplementary.csv b/tests/supplementary.csv new file mode 100644 index 0000000..9e40373 --- /dev/null +++ b/tests/supplementary.csv @@ -0,0 +1 @@ +Timestamp,Email Address,What is your GitHub username? \ No newline at end of file