diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 3042952..d5555ed 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -133,7 +133,7 @@ jobs: env: GITHUB_LOGIN: ${{ github.repository_owner }} GITHUB_OAUTH: ${{ secrets.RHUKI_READ_PAT }} - run: ./github-stats-*-runner collect-stats --organization=RedHat-Consulting-UK + run: ./github-stats-*-runner collect-stats --organization=RedHat-Consulting-UK --validate-org-config=false - name: Upload github-output.csv uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4 diff --git a/.gitignore b/.gitignore index 4ed7e27..8fe9be3 100644 --- a/.gitignore +++ b/.gitignore @@ -38,5 +38,6 @@ nb-configuration.xml # Local environment .env -logs/ creds.source +gh-members.csv +supplementary.csv diff --git a/README.md b/README.md index d99af66..3e194e9 100644 --- a/README.md +++ b/README.md @@ -9,19 +9,32 @@ Both JVM and Native mode are supported. ./mvnw clean install -Pnative ``` +Which allows you to run via: + +```bash +./target/github-stats-1.0.0-SNAPSHOT-runner +java -jar target/quarkus-app/quarkus-run.jar +``` + ## GitHub Auth Read permissions are required for the OAuth PAT. -``` +```bash export GITHUB_LOGIN=replace export GITHUB_OAUTH=replace ``` +## LDAP Lookup +```bash +ldapsearch -x -h ldap.corp.redhat.com -b dc=redhat,dc=com -s sub 'uid=gahealy' +``` + ## APIs Once you've built the code, you can execute by... -### CollectStatsService -``` +### CollectStats + +```bash ./target/github-stats-1.0.0-SNAPSHOT-runner collect-stats --organization={your-org} ``` @@ -31,9 +44,23 @@ Once the binary is complete, you can view the CSV: open github-output.csv ``` -### CreateWhoAreYouIssueService -`--members-csv` is a list of known members that have validated their GitHub ID against their RH ID. See: `tests/members.csv` as an example. +### CollectRedHatLdapSupplementaryList +Loop over the GitHub members and see if we can find them in LDAP. Output what we find to a CSV. +```bash +./target/github-stats-1.0.0-SNAPSHOT-runner collect-members-from-ldap --organization={your-org} --members-csv={list-of-known-members} --csv-output=supplementary.csv --fail-if-no-vpn=false ``` + +### GitHubMemberInRedHatLdap +Loop over the GitHub members and see if we can find them in LDAP. Output what we find to a CSV. + +```bash +./target/github-stats-1.0.0-SNAPSHOT-runner github-member-in-ldap --dry-run=true --organization={your-org} --issue-repo={a-repo-in-that-org} --members-csv={list-of-known-members} --supplementary-csv={list-of-supplementary-members} -fail-if-no-vpn=false +``` + +### CreateWhoAreYouIssue +`--members-csv` is a list of known members that have validated their GitHub ID against their RH ID. See: `tests/members.csv` as an example. + +```bash ./target/github-stats-1.0.0-SNAPSHOT-runner create-who-are-you-issues --dry-run=true --organization={your-org} --issue-repo={a-repo-in-that-org} --members-csv={list-of-known-members} --fail-if-no-vpn=false ``` \ No newline at end of file diff --git a/pom.xml b/pom.xml index 576dc86..7597d99 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ 1.0.0-SNAPSHOT 3.12.1 - 17 + 21 UTF-8 UTF-8 quarkus-bom @@ -16,6 +16,7 @@ true 3.2.3 + @@ -27,6 +28,7 @@ + io.quarkus @@ -61,7 +63,17 @@ io.quarkus quarkus-caffeine + + org.jboss.slf4j + slf4j-jboss-logmanager + + + io.quarkiverse.freemarker + quarkus-freemarker + 1.0.0 + + @@ -128,11 +140,9 @@ + -H:+UnlockExperimentalVMOptions,-H:ReflectionConfigurationFiles=reflection-config.json,-H:-UnlockExperimentalVMOptions false native - - -H:+UnlockExperimentalVMOptions,-H:ReflectionConfigurationFiles=reflection-config.json,-H:-UnlockExperimentalVMOptions - diff --git a/scripts/download-memebers-sheet.sh b/scripts/download-memebers-sheet.sh new file mode 100755 index 0000000..4f14aaf --- /dev/null +++ b/scripts/download-memebers-sheet.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +open https://docs.google.com/spreadsheets/d/1Aesp-sIoTvV-a0Qd-Kt0IpiU7fxvJLPDq8SwZ43vSOw/gviz/tq?tqx=out:csv + +sleep 5s +mv ~/Downloads/data.csv gh-members.csv \ No newline at end of file diff --git a/scripts/run-redhat-cop.sh b/scripts/run-redhat-cop.sh new file mode 100755 index 0000000..4f57cf1 --- /dev/null +++ b/scripts/run-redhat-cop.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +scripts/download-memebers-sheet.sh + +./mvnw clean install -Pnative + +#./target/github-stats-1.0.0-SNAPSHOT-runner collect-stats --organization=redhat-cop + +./target/github-stats-1.0.0-SNAPSHOT-runner collect-members-from-ldap --organization=redhat-cop --members-csv=gh-members.csv --csv-output=supplementary.csv --fail-if-no-vpn=true +./target/github-stats-1.0.0-SNAPSHOT-runner github-member-in-ldap --organization=redhat-cop --issue-repo=org --members-csv=gh-members.csv --supplementary-csv=supplementary.csv --fail-if-no-vpn=true \ No newline at end of file diff --git a/src/main/java/com/garethahealy/githubstats/GitHubStatsApplication.java b/src/main/java/com/garethahealy/githubstats/GitHubStatsApplication.java index 71edfb2..c747c49 100644 --- a/src/main/java/com/garethahealy/githubstats/GitHubStatsApplication.java +++ b/src/main/java/com/garethahealy/githubstats/GitHubStatsApplication.java @@ -1,8 +1,10 @@ package com.garethahealy.githubstats; -import com.garethahealy.githubstats.commands.CollectStatsCommand; -import com.garethahealy.githubstats.commands.CreateWhoAreYouIssueCommand; -import com.garethahealy.githubstats.commands.QuarkusCommand; +import com.garethahealy.githubstats.commands.GitHubStatsCommand; +import com.garethahealy.githubstats.commands.stats.CollectStatsCommand; +import com.garethahealy.githubstats.commands.users.CollectRedHatLdapSupplementaryListCommand; +import com.garethahealy.githubstats.commands.users.CreateWhoAreYouIssueCommand; +import com.garethahealy.githubstats.commands.users.GitHubMemberInRedHatLdapCommand; import io.quarkus.runtime.Quarkus; import io.quarkus.runtime.QuarkusApplication; import io.quarkus.runtime.annotations.QuarkusMain; @@ -15,6 +17,11 @@ public class GitHubStatsApplication implements QuarkusApplication { CollectStatsCommand collectStatsCommand; @Inject CreateWhoAreYouIssueCommand createWhoAreYouIssueCommand; + @Inject + GitHubMemberInRedHatLdapCommand gitHubMemberInRedHatLdapCommand; + + @Inject + CollectRedHatLdapSupplementaryListCommand collectRedHatLdapSupplementaryListCommand; public static void main(String[] args) { Quarkus.run(GitHubStatsApplication.class, args); @@ -22,9 +29,11 @@ public static void main(String[] args) { @Override public int run(String... args) throws Exception { - return new CommandLine(new QuarkusCommand()) + return new CommandLine(new GitHubStatsCommand()) .addSubcommand(collectStatsCommand) .addSubcommand(createWhoAreYouIssueCommand) + .addSubcommand(gitHubMemberInRedHatLdapCommand) + .addSubcommand(collectRedHatLdapSupplementaryListCommand) .execute(args); } } diff --git a/src/main/java/com/garethahealy/githubstats/commands/QuarkusCommand.java b/src/main/java/com/garethahealy/githubstats/commands/GitHubStatsCommand.java similarity index 87% rename from src/main/java/com/garethahealy/githubstats/commands/QuarkusCommand.java rename to src/main/java/com/garethahealy/githubstats/commands/GitHubStatsCommand.java index a22c5f7..56d76f9 100644 --- a/src/main/java/com/garethahealy/githubstats/commands/QuarkusCommand.java +++ b/src/main/java/com/garethahealy/githubstats/commands/GitHubStatsCommand.java @@ -6,5 +6,5 @@ name = "github-stats", description = "GitHub helper utility", subcommands = {CommandLine.HelpCommand.class}) -public class QuarkusCommand { +public class GitHubStatsCommand { } \ No newline at end of file diff --git a/src/main/java/com/garethahealy/githubstats/commands/CollectStatsCommand.java b/src/main/java/com/garethahealy/githubstats/commands/stats/CollectStatsCommand.java similarity index 79% rename from src/main/java/com/garethahealy/githubstats/commands/CollectStatsCommand.java rename to src/main/java/com/garethahealy/githubstats/commands/stats/CollectStatsCommand.java index 7ef9fa9..822f738 100644 --- a/src/main/java/com/garethahealy/githubstats/commands/CollectStatsCommand.java +++ b/src/main/java/com/garethahealy/githubstats/commands/stats/CollectStatsCommand.java @@ -1,11 +1,12 @@ -package com.garethahealy.githubstats.commands; +package com.garethahealy.githubstats.commands.stats; -import com.garethahealy.githubstats.rest.client.CollectStatsService; +import com.garethahealy.githubstats.services.stats.CollectStatsService; import jakarta.enterprise.context.Dependent; import jakarta.inject.Inject; import picocli.CommandLine; import java.io.IOException; +import java.util.concurrent.ExecutionException; @Dependent @CommandLine.Command(name = "collect-stats", mixinStandardHelpOptions = true, description = "Collect the stats in CSV format") @@ -14,7 +15,7 @@ public class CollectStatsCommand implements Runnable { @CommandLine.Option(names = {"-org", "--organization"}, description = "GitHub organization", required = true) String organization; - @CommandLine.Option(names = {"-cfg", "--validate-org-config"}, description = "Whether to check the 'org/config.yaml'", defaultValue = "false") + @CommandLine.Option(names = {"-cfg", "--validate-org-config"}, description = "Whether to check the 'org/config.yaml'", defaultValue = "true") boolean validateOrgConfig; @CommandLine.Option(names = {"-o", "--csv-output"}, description = "Output location for CSV", defaultValue = "github-output.csv") @@ -27,7 +28,7 @@ public class CollectStatsCommand implements Runnable { public void run() { try { collectStatsService.run(organization, validateOrgConfig, output); - } catch (IOException e) { + } catch (IOException | InterruptedException | ExecutionException e) { throw new RuntimeException(e); } } diff --git a/src/main/java/com/garethahealy/githubstats/commands/users/CollectRedHatLdapSupplementaryListCommand.java b/src/main/java/com/garethahealy/githubstats/commands/users/CollectRedHatLdapSupplementaryListCommand.java new file mode 100644 index 0000000..fe4a89b --- /dev/null +++ b/src/main/java/com/garethahealy/githubstats/commands/users/CollectRedHatLdapSupplementaryListCommand.java @@ -0,0 +1,39 @@ +package com.garethahealy.githubstats.commands.users; + +import com.garethahealy.githubstats.services.users.CollectRedHatLdapSupplementaryListService; +import freemarker.template.TemplateException; +import jakarta.enterprise.context.Dependent; +import jakarta.inject.Inject; +import org.apache.directory.api.ldap.model.exception.LdapException; +import picocli.CommandLine; + +import java.io.IOException; + +@Dependent +@CommandLine.Command(name = "collect-members-from-ldap", mixinStandardHelpOptions = true, description = "Creates a supplementary CSV containing members who have added their GitHub ID to LDAP") +public class CollectRedHatLdapSupplementaryListCommand implements Runnable { + + @CommandLine.Option(names = {"-org", "--organization"}, description = "GitHub organization", required = true) + String organization; + + @CommandLine.Option(names = {"-o", "--csv-output"}, description = "Output location for CSV", defaultValue = "github-output.csv") + String output; + + @CommandLine.Option(names = {"-i", "--members-csv"}, description = "CSV of current known members", required = true) + String membersCsv; + + @CommandLine.Option(names = {"-vpn", "--fail-if-no-vpn"}, description = "Throw an exception if can't connect to LDAP") + boolean failNoVpn; + + @Inject + CollectRedHatLdapSupplementaryListService collectRedHatLdapSupplementaryListService; + + @Override + public void run() { + try { + collectRedHatLdapSupplementaryListService.run(organization, output, membersCsv, failNoVpn); + } catch (IOException | LdapException | TemplateException e) { + throw new RuntimeException(e); + } + } +} \ No newline at end of file diff --git a/src/main/java/com/garethahealy/githubstats/commands/CreateWhoAreYouIssueCommand.java b/src/main/java/com/garethahealy/githubstats/commands/users/CreateWhoAreYouIssueCommand.java similarity index 89% rename from src/main/java/com/garethahealy/githubstats/commands/CreateWhoAreYouIssueCommand.java rename to src/main/java/com/garethahealy/githubstats/commands/users/CreateWhoAreYouIssueCommand.java index c9ae247..61e5404 100644 --- a/src/main/java/com/garethahealy/githubstats/commands/CreateWhoAreYouIssueCommand.java +++ b/src/main/java/com/garethahealy/githubstats/commands/users/CreateWhoAreYouIssueCommand.java @@ -1,6 +1,6 @@ -package com.garethahealy.githubstats.commands; +package com.garethahealy.githubstats.commands.users; -import com.garethahealy.githubstats.rest.client.CreateWhoAreYouIssueService; +import com.garethahealy.githubstats.services.users.CreateWhoAreYouIssueService; import jakarta.enterprise.context.Dependent; import jakarta.inject.Inject; import org.apache.directory.api.ldap.model.exception.LdapException; @@ -21,7 +21,7 @@ public class CreateWhoAreYouIssueCommand implements Runnable { @CommandLine.Option(names = {"-dry", "--dry-run"}, description = "Dry-run aka don't actually create the GitHub issues", required = true) boolean dryRun; - @CommandLine.Option(names = {"-i", "--members-csv"}, description = "CSV container current known members", required = true) + @CommandLine.Option(names = {"-i", "--members-csv"}, description = "CSV of current known members", required = true) String membersCsv; @CommandLine.Option(names = {"-vpn", "--fail-if-no-vpn"}, description = "Throw an exception if can't connect to LDAP") diff --git a/src/main/java/com/garethahealy/githubstats/commands/users/GitHubMemberInRedHatLdapCommand.java b/src/main/java/com/garethahealy/githubstats/commands/users/GitHubMemberInRedHatLdapCommand.java new file mode 100644 index 0000000..506ad2e --- /dev/null +++ b/src/main/java/com/garethahealy/githubstats/commands/users/GitHubMemberInRedHatLdapCommand.java @@ -0,0 +1,45 @@ +package com.garethahealy.githubstats.commands.users; + +import com.garethahealy.githubstats.services.users.GitHubMemberInRedHatLdapService; +import freemarker.template.TemplateException; +import jakarta.enterprise.context.Dependent; +import jakarta.inject.Inject; +import org.apache.directory.api.ldap.model.exception.LdapException; +import picocli.CommandLine; + +import java.io.IOException; + +@Dependent +@CommandLine.Command(name = "github-member-in-ldap", mixinStandardHelpOptions = true, description = "Creates a single issue containing any users that are part of the GitHub Org but not in LDAP") +public class GitHubMemberInRedHatLdapCommand implements Runnable { + + @CommandLine.Option(names = {"-org", "--organization"}, description = "GitHub organization", required = true) + String organization; + + @CommandLine.Option(names = {"-repo", "--issue-repo"}, description = "Repo where the issues should be created, i.e.: 'org'", required = true) + String orgRepo; + + @CommandLine.Option(names = {"-dry", "--dry-run"}, description = "Dry-run aka don't actually create the GitHub issue", required = true) + boolean dryRun; + + @CommandLine.Option(names = {"-i", "--members-csv"}, description = "CSV of current known members", required = true) + String membersCsv; + + @CommandLine.Option(names = {"-s", "--supplementary-csv"}, description = "CSV of current known members, generated by 'collect-members-from-ldap'", required = true) + String supplementaryCsv; + + @CommandLine.Option(names = {"-vpn", "--fail-if-no-vpn"}, description = "Throw an exception if can't connect to LDAP") + boolean failNoVpn; + + @Inject + GitHubMemberInRedHatLdapService gitHubMemberInRedHatLdapService; + + @Override + public void run() { + try { + gitHubMemberInRedHatLdapService.run(organization, orgRepo, dryRun, membersCsv, supplementaryCsv, failNoVpn); + } catch (IOException | LdapException | TemplateException e) { + throw new RuntimeException(e); + } + } +} \ No newline at end of file diff --git a/src/main/java/com/garethahealy/githubstats/model/MembersInfo.java b/src/main/java/com/garethahealy/githubstats/model/MembersInfo.java index 49a8d03..99ba496 100644 --- a/src/main/java/com/garethahealy/githubstats/model/MembersInfo.java +++ b/src/main/java/com/garethahealy/githubstats/model/MembersInfo.java @@ -1,9 +1,46 @@ package com.garethahealy.githubstats.model; +import io.quarkus.runtime.annotations.RegisterForReflection; + +import java.util.Arrays; +import java.util.List; + +@RegisterForReflection public class MembersInfo { public enum Headers { Timestamp, EmailAddress, WhatIsYourGitHubUsername } + + private final String timestamp; + private final String emailAddress; + private final String whatIsYourGitHubUsername; + private String redHatUserId; + + public String getEmailAddress() { + return emailAddress; + } + + public String getRedHatUserId() { + if (redHatUserId == null || redHatUserId.isEmpty()) { + redHatUserId = emailAddress.split("@")[0]; + } + + return redHatUserId; + } + + public String getWhatIsYourGitHubUsername() { + return whatIsYourGitHubUsername; + } + + public MembersInfo(String timestamp, String emailAddress, String whatIsYourGitHubUsername) { + this.timestamp = timestamp; + this.emailAddress = emailAddress; + this.whatIsYourGitHubUsername = whatIsYourGitHubUsername; + } + + public List toArray() { + return Arrays.asList(timestamp, emailAddress, whatIsYourGitHubUsername); + } } diff --git a/src/main/java/com/garethahealy/githubstats/model/RepoInfo.java b/src/main/java/com/garethahealy/githubstats/model/RepoInfo.java index 62f56c1..6322920 100644 --- a/src/main/java/com/garethahealy/githubstats/model/RepoInfo.java +++ b/src/main/java/com/garethahealy/githubstats/model/RepoInfo.java @@ -5,6 +5,7 @@ import java.io.IOException; import java.text.DateFormat; import java.text.SimpleDateFormat; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -71,12 +72,12 @@ public RepoInfo(String repoName, this.repoName = repoName; this.lastCommitAuthor = lastCommit == null || lastCommit.getAuthor() == null ? null : lastCommit.getAuthor().getLogin(); this.lastCommitDate = lastCommit == null ? null : df.format(lastCommit.getCommitDate()); - this.cop = topics.stream().filter(topic -> topic.contains("-cop") || topic.contains("gpte")).findFirst().orElse(null); + this.cop = topics == null ? null : topics.stream().filter(topic -> topic.contains("-cop") || topic.contains("gpte")).findFirst().orElse(null); this.contributorCount = contributors == null ? 0 : contributors.size(); this.commitCount = commits == null ? 0 : commits.size(); this.openIssueCount = issues == null ? 0 : issues.size(); this.openPullRequestCount = pullRequests == null ? 0 : pullRequests.size(); - this.topics = topics; + this.topics = topics == null ? new ArrayList<>() : topics; this.clonesInPast14Days = cloneTraffic == null ? 0 : cloneTraffic.getUniques(); this.viewsInPast14Days = viewTraffic == null ? 0 : viewTraffic.getUniques(); this.hasOwners = hasOwners; diff --git a/src/main/java/com/garethahealy/githubstats/rest/client/BaseGitHubService.java b/src/main/java/com/garethahealy/githubstats/rest/client/BaseGitHubService.java deleted file mode 100644 index 85bbd8e..0000000 --- a/src/main/java/com/garethahealy/githubstats/rest/client/BaseGitHubService.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.garethahealy.githubstats.rest.client; - -import jakarta.inject.Inject; -import org.jboss.logging.Logger; -import org.kohsuke.github.GitHub; -import org.kohsuke.github.GitHubBuilder; - -import java.io.IOException; - -public abstract class BaseGitHubService { - - @Inject - Logger logger; - - protected GitHub getGitHub() throws IOException { - logger.info("Starting..."); - - GitHub gitHub = GitHubBuilder.fromEnvironment().build(); - if (!gitHub.isCredentialValid()) { - throw new IllegalStateException("isCredentialValid - are GITHUB_LOGIN / GITHUB_OAUTH valid?"); - } - - if (gitHub.isAnonymous()) { - throw new IllegalStateException("isAnonymous - have you set GITHUB_LOGIN / GITHUB_OAUTH ?"); - } - - logger.infof("RateLimit: limit %s, remaining %s, resetDate %s", gitHub.getRateLimit().getLimit(), gitHub.getRateLimit().getRemaining(), gitHub.getRateLimit().getResetDate()); - if (gitHub.getRateLimit().getRemaining() == 0) { - throw new IllegalStateException("RateLimit - is zero, you need to wait until the reset date"); - } - - return gitHub; - } -} diff --git a/src/main/java/com/garethahealy/githubstats/rest/client/CollectStatsService.java b/src/main/java/com/garethahealy/githubstats/rest/client/CollectStatsService.java deleted file mode 100644 index fe09037..0000000 --- a/src/main/java/com/garethahealy/githubstats/rest/client/CollectStatsService.java +++ /dev/null @@ -1,172 +0,0 @@ -package com.garethahealy.githubstats.rest.client; - -import com.garethahealy.githubstats.model.RepoInfo; -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.inject.Inject; -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVPrinter; -import org.apache.commons.io.FileUtils; -import org.jboss.logging.Logger; -import org.kohsuke.github.*; - -import java.io.File; -import java.io.IOException; -import java.nio.charset.Charset; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.time.Duration; -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -@ApplicationScoped -public class CollectStatsService extends BaseGitHubService { - - @Inject - Logger logger; - - public List run(String organization, boolean validateOrgConfig, String output) throws IOException { - List answer = new ArrayList<>(); - - GitHub gitHub = getGitHub(); - GHOrganization org = gitHub.getOrganization(organization); - - String configContent = validateOrgConfig ? getOrgConfigYaml(org) : ""; - - CSVFormat csvFormat = CSVFormat.Builder.create(CSVFormat.DEFAULT).setHeader((RepoInfo.Headers.class)).build(); - LocalDateTime flushAt = LocalDateTime.now(); - try (CSVPrinter csvPrinter = new CSVPrinter(Files.newBufferedWriter(Paths.get(output)), csvFormat)) { - Map repos = org.getRepositories(); - logger.infof("Found %s repos.", repos.size()); - - int i = 1; - for (Map.Entry current : repos.entrySet()) { - logger.infof("Working on: %s - %s / %s", current.getValue().getName(), i, repos.size()); - - GHRepository repo = current.getValue(); - String repoName = repo.getName(); - - List contributors = null; - List commits = null; - List issues = null; - List pullRequests = null; - List topics = new ArrayList<>(); - GHCommit lastCommit = null; - GHRepositoryCloneTraffic cloneTraffic = null; - GHRepositoryViewTraffic viewTraffic = null; - boolean inConfig = configContent.contains(repoName); - boolean isArchived = repo.isArchived(); - boolean hasOwners = false; - boolean hasCodeOwners = false; - boolean hasWorkflows = false; - boolean hasTravis = false; - boolean hasRenovate = false; - - if (!isArchived) { - logger.info("-> listContributors"); - contributors = repo.listContributors().toList(); - - logger.info("-> listIssues"); - issues = repo.getIssues(GHIssueState.OPEN); - - logger.info("-> listPullRequests"); - pullRequests = repo.getPullRequests(GHIssueState.OPEN); - - logger.info("-> listTopics"); - topics = repo.listTopics(); - - try { - logger.info("-> listCommits"); - commits = repo.listCommits().toList(); - lastCommit = commits.get(0); - } catch (GHException | IOException ex) { - //ignore - has no commits - } - - try { - logger.info("-> Traffic"); - cloneTraffic = repo.getCloneTraffic(); - viewTraffic = repo.getViewTraffic(); - } catch (GHException | GHFileNotFoundException ex) { - //ignore - token doesn't have access to this repo to get traffic - } - - try { - logger.info("-> OWNERS"); - GHContent owners = repo.getFileContent("OWNERS"); - hasOwners = owners != null && owners.isFile(); - } catch (GHFileNotFoundException ex) { - //ignore - file doesn't exist - } - - try { - logger.info("-> CODEOWNERS"); - GHContent codeowners = repo.getFileContent("CODEOWNERS"); - hasCodeOwners = codeowners != null && codeowners.isFile(); - } catch (GHFileNotFoundException ex) { - //ignore - file doesn't exist - } - - try { - logger.info("-> .github/workflows"); - List workflows = repo.getDirectoryContent(".github/workflows"); - hasWorkflows = workflows != null && !workflows.isEmpty(); - } catch (GHFileNotFoundException ex) { - //ignore - file doesn't exist - } - - try { - logger.info("-> .travis.yml"); - GHContent travis = repo.getFileContent(".travis.yml"); - hasTravis = travis != null && travis.isFile(); - } catch (GHFileNotFoundException ex) { - //ignore - file doesn't exist - } - - try { - logger.info("-> renovate.json"); - GHContent renovate = repo.getFileContent("renovate.json"); - hasRenovate = renovate != null && renovate.isFile(); - } catch (GHFileNotFoundException ex) { - //ignore - file doesn't exist - } - } - - RepoInfo repoInfo = new RepoInfo(repoName, lastCommit, contributors, commits, issues, pullRequests, - topics, cloneTraffic, viewTraffic, hasOwners, hasCodeOwners, hasWorkflows, hasTravis, hasRenovate, inConfig, isArchived); - - answer.add(repoInfo); - - csvPrinter.printRecord(repoInfo.toArray()); - - logger.info("-> DONE"); - - if (Duration.between(flushAt, LocalDateTime.now()).getSeconds() > 60) { - flushAt = LocalDateTime.now(); - csvPrinter.flush(); - - logger.infof("RateLimit: limit %s, remaining %s, resetDate %s", gitHub.getRateLimit().getLimit(), gitHub.getRateLimit().getRemaining(), gitHub.getRateLimit().getResetDate()); - } - - i++; - } - } - - logger.info("Finished."); - logger.infof("RateLimit: limit %s, remaining %s, resetDate %s", gitHub.getRateLimit().getLimit(), gitHub.getRateLimit().getRemaining(), gitHub.getRateLimit().getResetDate()); - - return answer; - } - - private String getOrgConfigYaml(GHOrganization org) throws IOException { - logger.info("Downloading org/config.yaml"); - - GHRepository coreOrg = org.getRepository("org"); - GHContent orgConfig = coreOrg.getFileContent("config.yaml"); - File configOutputFile = new File("target/core-config.yaml"); - FileUtils.copyInputStreamToFile(orgConfig.read(), configOutputFile); - - return FileUtils.readFileToString(configOutputFile, Charset.defaultCharset()); - } -} diff --git a/src/main/java/com/garethahealy/githubstats/services/CsvService.java b/src/main/java/com/garethahealy/githubstats/services/CsvService.java new file mode 100644 index 0000000..dba589b --- /dev/null +++ b/src/main/java/com/garethahealy/githubstats/services/CsvService.java @@ -0,0 +1,37 @@ +package com.garethahealy.githubstats.services; + +import com.garethahealy.githubstats.model.MembersInfo; +import jakarta.enterprise.context.ApplicationScoped; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVRecord; + +import java.io.FileReader; +import java.io.IOException; +import java.io.Reader; +import java.util.HashMap; +import java.util.Map; + +@ApplicationScoped +public class CsvService { + + public Map getKnownMembers(String membersCsv) throws IOException { + Map answer = new HashMap<>(); + CSVFormat csvFormat = CSVFormat.Builder.create(CSVFormat.DEFAULT) + .setHeader(MembersInfo.Headers.class) + .setSkipHeaderRecord(true) + .build(); + + try (Reader in = new FileReader(membersCsv)) { + Iterable records = csvFormat.parse(in); + for (CSVRecord record : records) { + String timestamp = record.get(MembersInfo.Headers.Timestamp); + String redhatEmail = record.get(MembersInfo.Headers.EmailAddress); + String username = record.get(MembersInfo.Headers.WhatIsYourGitHubUsername); + + answer.put(username, new MembersInfo(timestamp, redhatEmail, username)); + } + } + + return answer; + } +} diff --git a/src/main/java/com/garethahealy/githubstats/services/GitHubService.java b/src/main/java/com/garethahealy/githubstats/services/GitHubService.java new file mode 100644 index 0000000..64434f4 --- /dev/null +++ b/src/main/java/com/garethahealy/githubstats/services/GitHubService.java @@ -0,0 +1,151 @@ +package com.garethahealy.githubstats.services; + +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import org.jboss.logging.Logger; +import org.kohsuke.github.*; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +@ApplicationScoped +public class GitHubService { + + @Inject + Logger logger; + + public GitHub getGitHub() throws IOException { + logger.info("Starting..."); + + GitHub gitHub = GitHubBuilder.fromEnvironment().build(); + if (!gitHub.isCredentialValid()) { + throw new IllegalStateException("isCredentialValid - are GITHUB_LOGIN / GITHUB_OAUTH valid?"); + } + + if (gitHub.isAnonymous()) { + throw new IllegalStateException("isAnonymous - have you set GITHUB_LOGIN / GITHUB_OAUTH ?"); + } + + logger.infof("RateLimit: limit %s, remaining %s, resetDate %s", gitHub.getRateLimit().getLimit(), gitHub.getRateLimit().getRemaining(), gitHub.getRateLimit().getResetDate()); + if (gitHub.getRateLimit().getRemaining() == 0) { + throw new IllegalStateException("RateLimit - is zero, you need to wait until the reset date"); + } + + return gitHub; + } + + public void logRateLimit(GitHub gitHub) throws IOException { + logger.infof("RateLimit: limit %s, remaining %s, resetDate %s", gitHub.getRateLimit().getLimit(), gitHub.getRateLimit().getRemaining(), gitHub.getRateLimit().getResetDate()); + } + + public GHOrganization getOrganization(GitHub gitHub, String organization) throws IOException { + return gitHub.getOrganization(organization); + } + + public Map getRepositories(GHOrganization org) throws IOException { + return org.getRepositories(); + } + + public GHRepository getRepository(GHOrganization org, String repo) throws IOException { + return org.getRepository(repo); + } + + public List listMembers(GHOrganization org) throws IOException { + return org.listMembers().toList(); + } + + public List listContributors(GHRepository repo) throws IOException { + logger.debugf("-> listContributors", repo.getName()); + return repo.listContributors().toList(); + } + + public List listOpenIssues(GHRepository repo) throws IOException { + logger.debugf("-> listOpenIssues", repo.getName()); + return repo.getIssues(GHIssueState.OPEN); + } + + public List listOpenPullRequests(GHRepository repo) throws IOException { + logger.debugf("-> listOpenPullRequests", repo.getName()); + return repo.getPullRequests(GHIssueState.OPEN); + } + + public List listTopics(GHRepository repo) throws IOException { + logger.debugf("-> listTopics", repo.getName()); + return repo.listTopics(); + } + + public List listCommits(GHRepository repo) { + List commits = null; + try { + logger.debugf("-> listCommits", repo.getName()); + commits = repo.listCommits().toList(); + } catch (GHException | IOException ex) { + //ignore - has no commits + logger.debug(ex); + } + return commits; + } + + public GHRepositoryCloneTraffic cloneTraffic(GHRepository repo) { + GHRepositoryCloneTraffic traffic = null; + try { + logger.debugf("-> cloneTraffic", repo.getName()); + traffic = repo.getCloneTraffic(); + } catch (GHException | IOException ex) { + //ignore - token doesn't have access to this repo to get traffic + logger.debug(ex); + } + + return traffic; + } + + public GHRepositoryViewTraffic viewTraffic(GHRepository repo) { + GHRepositoryViewTraffic traffic = null; + try { + logger.debugf("-> viewTraffic", repo.getName()); + traffic = repo.getViewTraffic(); + } catch (GHException | IOException ex) { + //ignore - token doesn't have access to this repo to get traffic + logger.debug(ex); + } + + return traffic; + } + + public boolean hasOwners(GHRepository repo) { + return hasFileContent(repo, "OWNERS"); + } + + public boolean hasCodeOwners(GHRepository repo) { + return hasFileContent(repo, "CODEOWNERS"); + } + + public boolean hasWorkflows(GHRepository repo) { + return hasFileContent(repo, ".github/workflows"); + } + + public boolean hasTravis(GHRepository repo) { + return hasFileContent(repo, ".travis.yml"); + } + + public boolean hasRenovate(GHRepository repo) { + return hasFileContent(repo, "renovate.json"); + } + + private boolean hasFileContent(GHRepository repo, String path) { + boolean answer = false; + + try { + logger.debugf("-> %s", path, repo.getName()); + + GHContent content = repo.getFileContent(path); + answer = content != null && content.isFile(); + } catch (IOException ex) { + //ignore - file doesn't exist + logger.debug(ex); + } + + return answer; + } +} diff --git a/src/main/java/com/garethahealy/githubstats/services/LdapService.java b/src/main/java/com/garethahealy/githubstats/services/LdapService.java new file mode 100644 index 0000000..78b70e0 --- /dev/null +++ b/src/main/java/com/garethahealy/githubstats/services/LdapService.java @@ -0,0 +1,95 @@ +package com.garethahealy.githubstats.services; + +import jakarta.annotation.PostConstruct; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import org.apache.directory.api.ldap.model.cursor.EntryCursor; +import org.apache.directory.api.ldap.model.entry.Attribute; +import org.apache.directory.api.ldap.model.entry.Entry; +import org.apache.directory.api.ldap.model.exception.LdapException; +import org.apache.directory.api.ldap.model.message.SearchScope; +import org.apache.directory.api.ldap.model.name.Dn; +import org.apache.directory.ldap.client.api.LdapConnection; +import org.apache.directory.ldap.client.api.LdapNetworkConnection; +import org.eclipse.microprofile.config.inject.ConfigProperty; +import org.jboss.logging.Logger; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicBoolean; + +@ApplicationScoped +public class LdapService { + + @Inject + Logger logger; + + @ConfigProperty(name = "redhat.ldap.dn") + String ldapDn; + + @ConfigProperty(name = "redhat.ldap.connection") + String ldapConnection; + + @ConfigProperty(name = "redhat.ldap.warmup-user") + String ldapWarmupUser; + + private final AtomicBoolean warmedUp = new AtomicBoolean(false); + private Dn systemDn; + + public boolean canConnect() { + return warmedUp.get(); + } + + @PostConstruct + void init() { + try { + systemDn = new Dn(ldapDn); + try (LdapConnection connection = open()) { + try (EntryCursor cursor = connection.search(systemDn, "(uid=" + ldapWarmupUser + ")", SearchScope.SUBTREE, "dn")) { + for (Entry entry : cursor) { + logger.infof("Warmup found %s", entry.getDn()); + warmedUp.set(true); + break; + } + } + } + } catch (IOException | LdapException e) { + logger.error("Failed to open connection to LDAP", e); + } + } + + public LdapConnection open() { + return new LdapNetworkConnection(ldapConnection); + } + + public boolean searchOnUser(LdapConnection connection, String uid) throws LdapException, IOException { + String filter = "(uid=" + uid + ")"; + String value = search(connection, filter, null); + return !value.isEmpty(); + } + + public String searchOnGitHub(LdapConnection connection, String githubId) throws LdapException, IOException { + String filter = "(rhatSocialURL=Github->https://github.com/" + githubId + ")"; + return search(connection, filter, "rhatPrimaryMail"); + } + + private String search(LdapConnection connection, String filter, String attribute) throws LdapException, IOException { + String value = ""; + try (EntryCursor cursor = connection.search(systemDn, filter, SearchScope.SUBTREE, attribute)) { + for (Entry entry : cursor) { + if (attribute == null) { + logger.infof("Found %s", filter); + value = entry.getDn().getName(); + } else { + Attribute foundAttribute = entry.get(attribute); + if (foundAttribute != null) { + logger.infof("Found %s - returning %s", filter, attribute); + value = foundAttribute.get().toString(); + } + } + break; + } + } + + return value; + } +} diff --git a/src/main/java/com/garethahealy/githubstats/services/stats/CollectStatsService.java b/src/main/java/com/garethahealy/githubstats/services/stats/CollectStatsService.java new file mode 100644 index 0000000..e409955 --- /dev/null +++ b/src/main/java/com/garethahealy/githubstats/services/stats/CollectStatsService.java @@ -0,0 +1,129 @@ +package com.garethahealy.githubstats.services.stats; + +import com.garethahealy.githubstats.model.RepoInfo; +import com.garethahealy.githubstats.services.GitHubService; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; +import org.apache.commons.io.FileUtils; +import org.jboss.logging.Logger; +import org.kohsuke.github.*; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +@ApplicationScoped +public class CollectStatsService { + + @Inject + Logger logger; + + private final GitHubService gitHubService; + + @Inject + public CollectStatsService(GitHubService gitHubService) { + this.gitHubService = gitHubService; + } + + public void run(String organization, boolean validateOrgConfig, String output) throws IOException, ExecutionException, InterruptedException { + GitHub gitHub = gitHubService.getGitHub(); + GHOrganization org = gitHubService.getOrganization(gitHub, organization); + GHRepository coreOrg = gitHubService.getRepository(org, "org"); + Map repos = gitHubService.getRepositories(org); + + String configContent = validateOrgConfig ? getOrgConfigYaml(coreOrg) : ""; + + logger.infof("Found %s repos.", repos.size()); + + CSVFormat csvFormat = CSVFormat.Builder.create(CSVFormat.DEFAULT) + .setHeader((RepoInfo.Headers.class)) + .build(); + + int cores = Runtime.getRuntime().availableProcessors() * 2; + try (CSVPrinter csvPrinter = new CSVPrinter(Files.newBufferedWriter(Paths.get(output)), csvFormat)) { + try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + List> futures = new ArrayList<>(); + for (Map.Entry current : repos.entrySet()) { + futures.add(executor.submit(() -> { + logger.infof("Working on: %s", current.getValue().getName()); + + GHRepository repo = current.getValue(); + String repoName = repo.getName(); + boolean isArchived = repo.isArchived(); + + List contributors = null; + List commits = null; + List issues = null; + List pullRequests = null; + List topics = null; + GHCommit lastCommit = null; + GHRepositoryCloneTraffic cloneTraffic = null; + GHRepositoryViewTraffic viewTraffic = null; + boolean inConfig = false; + boolean hasOwners = false; + boolean hasCodeOwners = false; + boolean hasWorkflows = false; + boolean hasTravis = false; + boolean hasRenovate = false; + + if (!isArchived) { + contributors = gitHubService.listContributors(repo); + issues = gitHubService.listOpenIssues(repo); + pullRequests = gitHubService.listOpenPullRequests(repo); + topics = gitHubService.listTopics(repo); + commits = gitHubService.listCommits(repo); + lastCommit = commits != null && !commits.isEmpty() ? commits.getFirst() : null; + cloneTraffic = gitHubService.cloneTraffic(repo); + viewTraffic = gitHubService.viewTraffic(repo); + hasOwners = gitHubService.hasOwners(repo); + hasCodeOwners = gitHubService.hasCodeOwners(repo); + hasWorkflows = gitHubService.hasWorkflows(repo); + hasTravis = gitHubService.hasTravis(repo); + hasRenovate = gitHubService.hasRenovate(repo); + inConfig = configContent.contains(repoName); + } + + return new RepoInfo(repoName, lastCommit, contributors, commits, issues, pullRequests, topics, cloneTraffic, viewTraffic, + hasOwners, hasCodeOwners, hasWorkflows, hasTravis, hasRenovate, inConfig, isArchived); + })); + + if (futures.size() == cores) { + for (Future future : futures) { + csvPrinter.printRecord(future.get().toArray()); + } + + csvPrinter.flush(); + futures.clear(); + + gitHubService.logRateLimit(gitHub); + } + } + + for (Future future : futures) { + csvPrinter.printRecord(future.get().toArray()); + } + } + } + } + + private String getOrgConfigYaml(GHRepository coreOrg) throws IOException { + logger.info("Downloading org/config.yaml"); + + GHContent orgConfig = coreOrg.getFileContent("config.yaml"); + File configOutputFile = new File("target/core-config.yaml"); + FileUtils.copyInputStreamToFile(orgConfig.read(), configOutputFile); + + return FileUtils.readFileToString(configOutputFile, Charset.defaultCharset()); + } +} diff --git a/src/main/java/com/garethahealy/githubstats/services/users/CollectRedHatLdapSupplementaryListService.java b/src/main/java/com/garethahealy/githubstats/services/users/CollectRedHatLdapSupplementaryListService.java new file mode 100644 index 0000000..079f591 --- /dev/null +++ b/src/main/java/com/garethahealy/githubstats/services/users/CollectRedHatLdapSupplementaryListService.java @@ -0,0 +1,73 @@ +package com.garethahealy.githubstats.services.users; + +import com.garethahealy.githubstats.model.MembersInfo; +import com.garethahealy.githubstats.services.CsvService; +import com.garethahealy.githubstats.services.GitHubService; +import com.garethahealy.githubstats.services.LdapService; +import freemarker.template.TemplateException; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; +import org.apache.directory.api.ldap.model.exception.LdapException; +import org.apache.directory.ldap.client.api.LdapConnection; +import org.jboss.logging.Logger; +import org.kohsuke.github.GHOrganization; +import org.kohsuke.github.GHUser; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; +import java.util.Map; + +@ApplicationScoped +public class CollectRedHatLdapSupplementaryListService { + + @Inject + Logger logger; + + private final GitHubService gitHubService; + private final LdapService ldapService; + private final CsvService csvService; + + @Inject + public CollectRedHatLdapSupplementaryListService(GitHubService gitHubService, CsvService csvService, LdapService ldapService) { + this.gitHubService = gitHubService; + this.csvService = csvService; + this.ldapService = ldapService; + } + + public void run(String organization, String output, String membersCsv, boolean failNoVpn) throws IOException, LdapException, TemplateException { + GHOrganization org = gitHubService.getOrganization(gitHubService.getGitHub(), organization); + List members = gitHubService.listMembers(org); + + Map knownMembers = csvService.getKnownMembers(membersCsv); + + CSVFormat csvFormat = CSVFormat.Builder.create(CSVFormat.DEFAULT) + .setHeader((MembersInfo.Headers.class)) + .build(); + + try (CSVPrinter csvPrinter = new CSVPrinter(Files.newBufferedWriter(Paths.get(output)), csvFormat)) { + if (ldapService.canConnect()) { + try (LdapConnection connection = ldapService.open()) { + for (GHUser user : members) { + if (!knownMembers.containsKey(user.getLogin())) { + String email = ldapService.searchOnGitHub(connection, user.getLogin()); + if (!email.isEmpty()) { + csvPrinter.printRecord(new MembersInfo("", email, user.getLogin()).toArray()); + } + } + } + + } + } else { + if (failNoVpn) { + throw new IOException("Unable to connect to LDAP. Are you on the VPN?"); + } + } + } + + logger.info("Finished."); + } +} diff --git a/src/main/java/com/garethahealy/githubstats/rest/client/CreateWhoAreYouIssueService.java b/src/main/java/com/garethahealy/githubstats/services/users/CreateWhoAreYouIssueService.java similarity index 90% rename from src/main/java/com/garethahealy/githubstats/rest/client/CreateWhoAreYouIssueService.java rename to src/main/java/com/garethahealy/githubstats/services/users/CreateWhoAreYouIssueService.java index db80fdd..3da9c10 100644 --- a/src/main/java/com/garethahealy/githubstats/rest/client/CreateWhoAreYouIssueService.java +++ b/src/main/java/com/garethahealy/githubstats/services/users/CreateWhoAreYouIssueService.java @@ -1,6 +1,7 @@ -package com.garethahealy.githubstats.rest.client; +package com.garethahealy.githubstats.services.users; import com.garethahealy.githubstats.model.MembersInfo; +import com.garethahealy.githubstats.services.GitHubService; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import org.apache.commons.csv.CSVFormat; @@ -25,13 +26,20 @@ import java.util.Set; @ApplicationScoped -public class CreateWhoAreYouIssueService extends BaseGitHubService { +public class CreateWhoAreYouIssueService { @Inject Logger logger; + private final GitHubService gitHubService; + + @Inject + public CreateWhoAreYouIssueService(GitHubService gitHubService) { + this.gitHubService = gitHubService; + } + public void run(String organization, String issueRepo, boolean isDryRun, String membersCsv, boolean failNoVpn) throws IOException, LdapException { - GitHub gitHub = getGitHub(); + GitHub gitHub = gitHubService.getGitHub(); GHOrganization org = gitHub.getOrganization(organization); GHRepository orgRepo = org.getRepository(issueRepo); @@ -58,6 +66,7 @@ public void run(String organization, String issueRepo, boolean isDryRun, String if (isDryRun) { logger.infof("DRY-RUN: Would have created issue for %s in %s", current.getLogin(), orgRepo.getName()); } else { + //TODO: check if issue already exists builder.create(); logger.infof("Created issue for %s", current.getLogin()); @@ -66,6 +75,7 @@ public void run(String organization, String issueRepo, boolean isDryRun, String } logger.info("Issues DONE"); + logger.infof("RateLimit: limit %s, remaining %s, resetDate %s", gitHub.getRateLimit().getLimit(), gitHub.getRateLimit().getRemaining(), gitHub.getRateLimit().getResetDate()); Set membersLogins = getMembersLogins(members); for (String current : usernamesToIgnore) { diff --git a/src/main/java/com/garethahealy/githubstats/services/users/GitHubMemberInRedHatLdapService.java b/src/main/java/com/garethahealy/githubstats/services/users/GitHubMemberInRedHatLdapService.java new file mode 100644 index 0000000..ab2268c --- /dev/null +++ b/src/main/java/com/garethahealy/githubstats/services/users/GitHubMemberInRedHatLdapService.java @@ -0,0 +1,132 @@ +package com.garethahealy.githubstats.services.users; + +import com.garethahealy.githubstats.model.MembersInfo; +import com.garethahealy.githubstats.services.CsvService; +import com.garethahealy.githubstats.services.GitHubService; +import com.garethahealy.githubstats.services.LdapService; +import freemarker.template.Template; +import freemarker.template.TemplateException; +import io.quarkiverse.freemarker.TemplatePath; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import org.apache.directory.api.ldap.model.exception.LdapException; +import org.apache.directory.ldap.client.api.LdapConnection; +import org.jboss.logging.Logger; +import org.kohsuke.github.GHIssue; +import org.kohsuke.github.GHOrganization; +import org.kohsuke.github.GHRepository; +import org.kohsuke.github.GHUser; + +import java.io.IOException; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@ApplicationScoped +public class GitHubMemberInRedHatLdapService { + + @Inject + Logger logger; + + @Inject + @TemplatePath("GitHubMemberInRedHatLdap.ftl") + Template issue; + + private final GitHubService gitHubService; + private final LdapService ldapService; + private final CsvService csvService; + + @Inject + public GitHubMemberInRedHatLdapService(GitHubService gitHubService, CsvService csvService, LdapService ldapService) { + this.gitHubService = gitHubService; + this.csvService = csvService; + this.ldapService = ldapService; + } + + public void run(String organization, String issueRepo, boolean isDryRun, String membersCsv, String supplementaryCsv, boolean failNoVpn) throws IOException, LdapException, TemplateException { + GHOrganization org = gitHubService.getOrganization(gitHubService.getGitHub(), organization); + GHRepository orgRepo = gitHubService.getRepository(org, issueRepo); + List members = gitHubService.listMembers(org); + + Map knownMembers = csvService.getKnownMembers(membersCsv); + Map supplementaryMembers = csvService.getKnownMembers(supplementaryCsv); + + logger.infof("There are %s GitHub members", members.size()); + logger.infof("There are %s known members and %s supplementary members in the CSVs", knownMembers.size(), supplementaryMembers.size()); + + List ldapCheck = collectLdapCheckList(members, knownMembers, supplementaryMembers); + List usersToRemove = searchFor(ldapCheck, failNoVpn); + + createIssue(usersToRemove, orgRepo, isDryRun); + + logger.info("Finished."); + } + + private List collectLdapCheckList(List members, Map knownMembers, Map supplementaryMembers) { + List answer = new ArrayList<>(); + for (GHUser current : members) { + if (knownMembers.containsKey(current.getLogin())) { + logger.infof("Adding %s to LDAP check list from known members", current.getLogin()); + + answer.add(knownMembers.get(current.getLogin())); + } else { + if (supplementaryMembers.containsKey(current.getLogin())) { + logger.infof("Adding %s to LDAP check list from supplementary", current.getLogin()); + + answer.add(supplementaryMembers.get(current.getLogin())); + } + } + } + + logger.info("--> User Lookup DONE"); + return answer; + } + + private List searchFor(List ldapCheck, boolean failNoVpn) throws IOException, LdapException { + List answer = new ArrayList<>(); + if (ldapService.canConnect()) { + try (LdapConnection connection = ldapService.open()) { + for (MembersInfo current : ldapCheck) { + boolean found = ldapService.searchOnUser(connection, current.getRedHatUserId()); + if (!found) { + logger.warnf("Did not find %s in LDAP", current.getRedHatUserId()); + answer.add(current); + } + } + } + } else { + if (failNoVpn) { + throw new IOException("Unable to connect to LDAP. Are you on the VPN?"); + } + } + + logger.info("--> LDAP Lookup DONE"); + return answer; + } + + private void createIssue(List usersToRemove, GHRepository orgRepo, boolean isDryRun) throws TemplateException, IOException { + if (!usersToRemove.isEmpty()) { + Map root = new HashMap<>(); + root.put("users", usersToRemove); + + StringWriter stringWriter = new StringWriter(); + issue.process(root, stringWriter); + + if (isDryRun) { + logger.infof("DRY-RUN: Would have created issue in %s", orgRepo.getName()); + logger.infof(stringWriter.toString()); + } else { + GHIssue createdIssue = orgRepo.createIssue("Remove users - Not in RH LDAP") + .label("admin") + .body(stringWriter.toString()) + .create(); + + logger.infof("Created issue: %s", createdIssue.getUrl()); + } + } + + logger.info("--> Issue creation DONE"); + } +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index e69de29..709e2d4 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -0,0 +1,3 @@ +redhat.ldap.dn=dc=redhat,dc=com +redhat.ldap.connection=ldap.corp.redhat.com +redhat.ldap.warmup-user=gahealy \ No newline at end of file diff --git a/src/main/resources/freemarker/templates/CreateWhoAreYouIssue.ftl b/src/main/resources/freemarker/templates/CreateWhoAreYouIssue.ftl new file mode 100644 index 0000000..90f8b7f --- /dev/null +++ b/src/main/resources/freemarker/templates/CreateWhoAreYouIssue.ftl @@ -0,0 +1,16 @@ +To be a member of the Red Hat CoP GitHub organization, you are required to be a Red Hat employee. +Non-employees are invited to be outside-collaborators (https://github.com/orgs/redhat-cop/outside-collaborators). + +To resolve GitHub IDs to Red Hat IDs, we check if a response of the below form has been provided, if not, we search LDAP. +- https://red.ht/github-redhat-cop-username + +If you are unsure how to set your GitHub ID within LDAP, see: +- https://source.redhat.com/departments/it/it-information-security/wiki/details_about_rover_github_information_security_and_scanning + +The below list of members have *${permissions}* and cannot be found using the above methods. + +Please complete the above form or add your GitHub handle to Rover. + +<#list users as user> +- @${user.getWhatIsYourGitHubUsername()} + \ No newline at end of file diff --git a/src/main/resources/freemarker/templates/GitHubMemberInRedHatLdap.ftl b/src/main/resources/freemarker/templates/GitHubMemberInRedHatLdap.ftl new file mode 100644 index 0000000..7e3e50a --- /dev/null +++ b/src/main/resources/freemarker/templates/GitHubMemberInRedHatLdap.ftl @@ -0,0 +1,5 @@ +The following users can no longer be found in LDAP: + +<#list users as user> +- @${user.getWhatIsYourGitHubUsername()} (${user.getEmailAddress()}) + \ No newline at end of file