Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
added ldap search command
Browse files Browse the repository at this point in the history
updated collect stats to use virtual threads to improve performance
garethahealy committed Jan 8, 2024
1 parent 4d4b3fb commit 3aff246
Showing 26 changed files with 865 additions and 234 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/build.yaml
Original file line number Diff line number Diff line change
@@ -133,7 +133,7 @@ jobs:
env:
GITHUB_LOGIN: ${{ github.repository_owner }}
GITHUB_OAUTH: ${{ secrets.RHUKI_READ_PAT }}
run: ./github-stats-*-runner collect-stats --organization=RedHat-Consulting-UK
run: ./github-stats-*-runner collect-stats --organization=RedHat-Consulting-UK --validate-org-config=false

- name: Upload github-output.csv
uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -38,5 +38,6 @@ nb-configuration.xml
# Local environment
.env

logs/
creds.source
gh-members.csv
supplementary.csv
37 changes: 32 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
@@ -9,19 +9,32 @@ Both JVM and Native mode are supported.
./mvnw clean install -Pnative
```

Which allows you to run via:

```bash
./target/github-stats-1.0.0-SNAPSHOT-runner
java -jar target/quarkus-app/quarkus-run.jar
```

## GitHub Auth
Read permissions are required for the OAuth PAT.

```
```bash
export GITHUB_LOGIN=replace
export GITHUB_OAUTH=replace
```

## LDAP Lookup
```bash
ldapsearch -x -h ldap.corp.redhat.com -b dc=redhat,dc=com -s sub 'uid=gahealy'
```

## APIs
Once you've built the code, you can execute by...

### CollectStatsService
```
### CollectStats

```bash
./target/github-stats-1.0.0-SNAPSHOT-runner collect-stats --organization={your-org}
```

@@ -31,9 +44,23 @@ Once the binary is complete, you can view the CSV:
open github-output.csv
```

### CreateWhoAreYouIssueService
`--members-csv` is a list of known members that have validated their GitHub ID against their RH ID. See: `tests/members.csv` as an example.
### CollectRedHatLdapSupplementaryList
Loop over the GitHub members and see if we can find them in LDAP. Output what we find to a CSV.

```bash
./target/github-stats-1.0.0-SNAPSHOT-runner collect-members-from-ldap --organization={your-org} --members-csv={list-of-known-members} --csv-output=supplementary.csv --fail-if-no-vpn=false
```

### GitHubMemberInRedHatLdap
Loop over the GitHub members and see if we can find them in LDAP. Output what we find to a CSV.

```bash
./target/github-stats-1.0.0-SNAPSHOT-runner github-member-in-ldap --dry-run=true --organization={your-org} --issue-repo={a-repo-in-that-org} --members-csv={list-of-known-members} --supplementary-csv={list-of-supplementary-members} -fail-if-no-vpn=false
```

### CreateWhoAreYouIssue
`--members-csv` is a list of known members that have validated their GitHub ID against their RH ID. See: `tests/members.csv` as an example.

```bash
./target/github-stats-1.0.0-SNAPSHOT-runner create-who-are-you-issues --dry-run=true --organization={your-org} --issue-repo={a-repo-in-that-org} --members-csv={list-of-known-members} --fail-if-no-vpn=false
```
18 changes: 14 additions & 4 deletions pom.xml
Original file line number Diff line number Diff line change
@@ -7,7 +7,7 @@
<version>1.0.0-SNAPSHOT</version>
<properties>
<compiler-plugin.version>3.12.1</compiler-plugin.version>
<maven.compiler.release>17</maven.compiler.release>
<maven.compiler.release>21</maven.compiler.release>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<quarkus.platform.artifact-id>quarkus-bom</quarkus.platform.artifact-id>
@@ -16,6 +16,7 @@
<skipITs>true</skipITs>
<surefire-plugin.version>3.2.3</surefire-plugin.version>
</properties>

<dependencyManagement>
<dependencies>
<dependency>
@@ -27,6 +28,7 @@
</dependency>
</dependencies>
</dependencyManagement>

<dependencies>
<dependency>
<groupId>io.quarkus</groupId>
@@ -61,7 +63,17 @@
<groupId>io.quarkus</groupId>
<artifactId>quarkus-caffeine</artifactId>
</dependency>
<dependency>
<groupId>org.jboss.slf4j</groupId>
<artifactId>slf4j-jboss-logmanager</artifactId>
</dependency>
<dependency>
<groupId>io.quarkiverse.freemarker</groupId>
<artifactId>quarkus-freemarker</artifactId>
<version>1.0.0</version>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
@@ -128,11 +140,9 @@
</property>
</activation>
<properties>
<quarkus.native.additional-build-args>-H:+UnlockExperimentalVMOptions,-H:ReflectionConfigurationFiles=reflection-config.json,-H:-UnlockExperimentalVMOptions</quarkus.native.additional-build-args>
<skipITs>false</skipITs>
<quarkus.package.type>native</quarkus.package.type>
<quarkus.native.additional-build-args>
-H:+UnlockExperimentalVMOptions,-H:ReflectionConfigurationFiles=reflection-config.json,-H:-UnlockExperimentalVMOptions
</quarkus.native.additional-build-args>
</properties>
</profile>
</profiles>
6 changes: 6 additions & 0 deletions scripts/download-memebers-sheet.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
#!/usr/bin/env bash

open https://docs.google.com/spreadsheets/d/1Aesp-sIoTvV-a0Qd-Kt0IpiU7fxvJLPDq8SwZ43vSOw/gviz/tq?tqx=out:csv

sleep 5s
mv ~/Downloads/data.csv gh-members.csv
10 changes: 10 additions & 0 deletions scripts/run-redhat-cop.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
#!/usr/bin/env bash

scripts/download-memebers-sheet.sh

./mvnw clean install -Pnative

#./target/github-stats-1.0.0-SNAPSHOT-runner collect-stats --organization=redhat-cop

./target/github-stats-1.0.0-SNAPSHOT-runner collect-members-from-ldap --organization=redhat-cop --members-csv=gh-members.csv --csv-output=supplementary.csv --fail-if-no-vpn=true
./target/github-stats-1.0.0-SNAPSHOT-runner github-member-in-ldap --organization=redhat-cop --issue-repo=org --members-csv=gh-members.csv --supplementary-csv=supplementary.csv --fail-if-no-vpn=true
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
package com.garethahealy.githubstats;

import com.garethahealy.githubstats.commands.CollectStatsCommand;
import com.garethahealy.githubstats.commands.CreateWhoAreYouIssueCommand;
import com.garethahealy.githubstats.commands.QuarkusCommand;
import com.garethahealy.githubstats.commands.GitHubStatsCommand;
import com.garethahealy.githubstats.commands.stats.CollectStatsCommand;
import com.garethahealy.githubstats.commands.users.CollectRedHatLdapSupplementaryListCommand;
import com.garethahealy.githubstats.commands.users.CreateWhoAreYouIssueCommand;
import com.garethahealy.githubstats.commands.users.GitHubMemberInRedHatLdapCommand;
import io.quarkus.runtime.Quarkus;
import io.quarkus.runtime.QuarkusApplication;
import io.quarkus.runtime.annotations.QuarkusMain;
@@ -15,16 +17,23 @@ public class GitHubStatsApplication implements QuarkusApplication {
CollectStatsCommand collectStatsCommand;
@Inject
CreateWhoAreYouIssueCommand createWhoAreYouIssueCommand;
@Inject
GitHubMemberInRedHatLdapCommand gitHubMemberInRedHatLdapCommand;

@Inject
CollectRedHatLdapSupplementaryListCommand collectRedHatLdapSupplementaryListCommand;

public static void main(String[] args) {
Quarkus.run(GitHubStatsApplication.class, args);
}

@Override
public int run(String... args) throws Exception {
return new CommandLine(new QuarkusCommand())
return new CommandLine(new GitHubStatsCommand())
.addSubcommand(collectStatsCommand)
.addSubcommand(createWhoAreYouIssueCommand)
.addSubcommand(gitHubMemberInRedHatLdapCommand)
.addSubcommand(collectRedHatLdapSupplementaryListCommand)
.execute(args);
}
}
Original file line number Diff line number Diff line change
@@ -6,5 +6,5 @@
name = "github-stats",
description = "GitHub helper utility",
subcommands = {CommandLine.HelpCommand.class})
public class QuarkusCommand {
public class GitHubStatsCommand {
}
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
package com.garethahealy.githubstats.commands;
package com.garethahealy.githubstats.commands.stats;

import com.garethahealy.githubstats.rest.client.CollectStatsService;
import com.garethahealy.githubstats.services.stats.CollectStatsService;
import jakarta.enterprise.context.Dependent;
import jakarta.inject.Inject;
import picocli.CommandLine;

import java.io.IOException;
import java.util.concurrent.ExecutionException;

@Dependent
@CommandLine.Command(name = "collect-stats", mixinStandardHelpOptions = true, description = "Collect the stats in CSV format")
@@ -14,7 +15,7 @@ public class CollectStatsCommand implements Runnable {
@CommandLine.Option(names = {"-org", "--organization"}, description = "GitHub organization", required = true)
String organization;

@CommandLine.Option(names = {"-cfg", "--validate-org-config"}, description = "Whether to check the 'org/config.yaml'", defaultValue = "false")
@CommandLine.Option(names = {"-cfg", "--validate-org-config"}, description = "Whether to check the 'org/config.yaml'", defaultValue = "true")
boolean validateOrgConfig;

@CommandLine.Option(names = {"-o", "--csv-output"}, description = "Output location for CSV", defaultValue = "github-output.csv")
@@ -27,7 +28,7 @@ public class CollectStatsCommand implements Runnable {
public void run() {
try {
collectStatsService.run(organization, validateOrgConfig, output);
} catch (IOException e) {
} catch (IOException | InterruptedException | ExecutionException e) {
throw new RuntimeException(e);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package com.garethahealy.githubstats.commands.users;

import com.garethahealy.githubstats.services.users.CollectRedHatLdapSupplementaryListService;
import freemarker.template.TemplateException;
import jakarta.enterprise.context.Dependent;
import jakarta.inject.Inject;
import org.apache.directory.api.ldap.model.exception.LdapException;
import picocli.CommandLine;

import java.io.IOException;

@Dependent
@CommandLine.Command(name = "collect-members-from-ldap", mixinStandardHelpOptions = true, description = "Creates a supplementary CSV containing members who have added their GitHub ID to LDAP")
public class CollectRedHatLdapSupplementaryListCommand implements Runnable {

@CommandLine.Option(names = {"-org", "--organization"}, description = "GitHub organization", required = true)
String organization;

@CommandLine.Option(names = {"-o", "--csv-output"}, description = "Output location for CSV", defaultValue = "github-output.csv")
String output;

@CommandLine.Option(names = {"-i", "--members-csv"}, description = "CSV of current known members", required = true)
String membersCsv;

@CommandLine.Option(names = {"-vpn", "--fail-if-no-vpn"}, description = "Throw an exception if can't connect to LDAP")
boolean failNoVpn;

@Inject
CollectRedHatLdapSupplementaryListService collectRedHatLdapSupplementaryListService;

@Override
public void run() {
try {
collectRedHatLdapSupplementaryListService.run(organization, output, membersCsv, failNoVpn);
} catch (IOException | LdapException | TemplateException e) {
throw new RuntimeException(e);
}
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package com.garethahealy.githubstats.commands;
package com.garethahealy.githubstats.commands.users;

import com.garethahealy.githubstats.rest.client.CreateWhoAreYouIssueService;
import com.garethahealy.githubstats.services.users.CreateWhoAreYouIssueService;
import jakarta.enterprise.context.Dependent;
import jakarta.inject.Inject;
import org.apache.directory.api.ldap.model.exception.LdapException;
@@ -21,7 +21,7 @@ public class CreateWhoAreYouIssueCommand implements Runnable {
@CommandLine.Option(names = {"-dry", "--dry-run"}, description = "Dry-run aka don't actually create the GitHub issues", required = true)
boolean dryRun;

@CommandLine.Option(names = {"-i", "--members-csv"}, description = "CSV container current known members", required = true)
@CommandLine.Option(names = {"-i", "--members-csv"}, description = "CSV of current known members", required = true)
String membersCsv;

@CommandLine.Option(names = {"-vpn", "--fail-if-no-vpn"}, description = "Throw an exception if can't connect to LDAP")
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package com.garethahealy.githubstats.commands.users;

import com.garethahealy.githubstats.services.users.GitHubMemberInRedHatLdapService;
import freemarker.template.TemplateException;
import jakarta.enterprise.context.Dependent;
import jakarta.inject.Inject;
import org.apache.directory.api.ldap.model.exception.LdapException;
import picocli.CommandLine;

import java.io.IOException;

@Dependent
@CommandLine.Command(name = "github-member-in-ldap", mixinStandardHelpOptions = true, description = "Creates a single issue containing any users that are part of the GitHub Org but not in LDAP")
public class GitHubMemberInRedHatLdapCommand implements Runnable {

@CommandLine.Option(names = {"-org", "--organization"}, description = "GitHub organization", required = true)
String organization;

@CommandLine.Option(names = {"-repo", "--issue-repo"}, description = "Repo where the issues should be created, i.e.: 'org'", required = true)
String orgRepo;

@CommandLine.Option(names = {"-dry", "--dry-run"}, description = "Dry-run aka don't actually create the GitHub issue", required = true)
boolean dryRun;

@CommandLine.Option(names = {"-i", "--members-csv"}, description = "CSV of current known members", required = true)
String membersCsv;

@CommandLine.Option(names = {"-s", "--supplementary-csv"}, description = "CSV of current known members, generated by 'collect-members-from-ldap'", required = true)
String supplementaryCsv;

@CommandLine.Option(names = {"-vpn", "--fail-if-no-vpn"}, description = "Throw an exception if can't connect to LDAP")
boolean failNoVpn;

@Inject
GitHubMemberInRedHatLdapService gitHubMemberInRedHatLdapService;

@Override
public void run() {
try {
gitHubMemberInRedHatLdapService.run(organization, orgRepo, dryRun, membersCsv, supplementaryCsv, failNoVpn);
} catch (IOException | LdapException | TemplateException e) {
throw new RuntimeException(e);
}
}
}
37 changes: 37 additions & 0 deletions src/main/java/com/garethahealy/githubstats/model/MembersInfo.java
Original file line number Diff line number Diff line change
@@ -1,9 +1,46 @@
package com.garethahealy.githubstats.model;

import io.quarkus.runtime.annotations.RegisterForReflection;

import java.util.Arrays;
import java.util.List;

@RegisterForReflection
public class MembersInfo {
public enum Headers {
Timestamp,
EmailAddress,
WhatIsYourGitHubUsername
}

private final String timestamp;
private final String emailAddress;
private final String whatIsYourGitHubUsername;
private String redHatUserId;

public String getEmailAddress() {
return emailAddress;
}

public String getRedHatUserId() {
if (redHatUserId == null || redHatUserId.isEmpty()) {
redHatUserId = emailAddress.split("@")[0];
}

return redHatUserId;
}

public String getWhatIsYourGitHubUsername() {
return whatIsYourGitHubUsername;
}

public MembersInfo(String timestamp, String emailAddress, String whatIsYourGitHubUsername) {
this.timestamp = timestamp;
this.emailAddress = emailAddress;
this.whatIsYourGitHubUsername = whatIsYourGitHubUsername;
}

public List<String> toArray() {
return Arrays.asList(timestamp, emailAddress, whatIsYourGitHubUsername);
}
}
Loading

0 comments on commit 3aff246

Please sign in to comment.