Count with all sorts of contributions

[whyrusleeping]

Not sure whats wrong exactly, but the code in this repo doesnt produce the right output (its missing a lot of people).

Heres the go code i wrote that seems to be correct:

package main

import (
        "context"
        "encoding/json"
        "fmt"
        "github.com/google/go-github/github"
        "golang.org/x/oauth2"
        "io/ioutil"
        "net/http"
        "os"
        "strings"
)

var authtok = "YOUR GITHUB AUTH TOKEN"
var ghurl = "https://api.github.com"

var cli = oauth2.NewClient(context.Background(), oauth2.StaticTokenSource(&oauth2.Token{AccessToken: authtok}))

func getBlob(res string) ([]byte, error) {
        fname := "data/" + strings.Replace(res, "/", "-", -1)
        if _, err := os.Stat(fname); err == nil {
                return ioutil.ReadFile(fname)
        }

        req, err := http.NewRequest("GET", ghurl+res, nil)
        if err != nil {
                panic(err)
        }

        resp, err := cli.Do(req)
        if err != nil {
                return nil, err
        }

        if resp.StatusCode != 200 {
                return nil, fmt.Errorf("%s failed: %s", res, resp.Status)
        }

        data, err := ioutil.ReadAll(resp.Body)
        if err != nil {
                return nil, err
        }

        err = ioutil.WriteFile(fname, data, 0644)
        if err != nil {
                return nil, err
        }

        return data, nil
}

func getAllRepos(org string) []github.Repository {
        var repos []github.Repository
        for i := 1; i < 100; i++ {
                data, err := getBlob(fmt.Sprintf("/orgs/%s/repos?page=%d&per_page=100", org, i))
                if err != nil {
                        panic(err)
                }

                var these []github.Repository
                if err := json.Unmarshal(data, &these); err != nil {
                        panic(err)
                }
                repos = append(repos, these...)
                if len(these) < 100 {
                        break
                }
        }
        return repos
}

func getAllPRs(repo github.Repository) []github.PullRequest {
        var all []github.PullRequest
        for i := 1; i < 100; i++ {
                data, err := getBlob(fmt.Sprintf("/repos/%s/%s/pulls?page=%d&per_page=100&state=all", "ipfs", repo.GetName(), i))
                if err != nil {
                        panic(err)
                }

                var these []github.PullRequest
                if err := json.Unmarshal(data, &these); err != nil {
                        panic(err)
                }
                all = append(all, these...)

                if len(these) < 100 {
                        break
                }
        }

        return all
}

func getAllIssues(org string, repo github.Repository) []github.Issue {
        var all []github.Issue
        for i := 1; i < 100; i++ {
                data, err := getBlob(fmt.Sprintf("/repos/%s/%s/issues?page=%d&per_page=100&state=all", org, repo.GetName(), i))
                if err != nil {
                        panic(err)
                }

                var these []github.Issue
                if err := json.Unmarshal(data, &these); err != nil {
                        panic(err)
                }
                all = append(all, these...)

                if len(these) < 100 {
                        break
                }
        }

        return all
}

func getAllComments(org, repo string, issue github.Issue) []github.IssueComment {
        var all []github.IssueComment
        for i := 1; i < 100; i++ {
                data, err := getBlob(fmt.Sprintf("/repos/%s/%s/issues/%d/comments?page=%d&per_page=100&state=all", org, repo, issue.GetNumber(), i))
                if err != nil {
                        panic(err)
                }

                var these []github.IssueComment
                if err := json.Unmarshal(data, &these); err != nil {
                        panic(err)
                }
                all = append(all, these...)

                if len(these) < 100 {
                        break
                }
        }

        return all
}

type output struct {
        Url      string `json:"url"`
        Photo    string `json:"photo"`
        Username string `json:"username"`
}

func main() {
        allpeople := make(map[string]*github.User)
        for _, org := range []string{"ipfs", "ipld", "multiformats", "libp2p", "orbitdb"} {
                repos := getAllRepos(org)

                for _, repo := range repos {
                        issues := getAllIssues(org, repo)
                        for _, issue := range issues {
                                issue.Repository = &repo
                                allpeople[*issue.User.Login] = issue.User
                                comments := getAllComments(org, repo.GetName(), issue)
                                for _, c := range comments {
                                        allpeople[c.User.GetLogin()] = c.User
                                }
                        }
                }
        }

        var out []output
        for k, v := range allpeople {
                out = append(out, output{
                        Url:      v.GetHTMLURL(),
                        Photo:    v.GetAvatarURL(),
                        Username: k,
                })
        }
        enc := json.NewEncoder(os.Stdout)
        enc.SetIndent("", "  ")
        if err := enc.Encode(out); err != nil {
                panic(err)
        }
}

[daviddias]

Got it, @RichardLitt went for all contributors on repos (code contributors) -- https://github.com/ipfs/get-gh-contributors/blob/master/src/fetch.js#L37 -- while you went to a more board definition of a contributor. Makes sense to me :) I'm not sure if it was a decision at the time to only check code commiters, but new version looks better. woot!

[whyrusleeping]

We previously talked about counting people filing issues and commenting on issues as contributors too, which is the approach i took

[RichardLitt]

Yep. That was the approach for name-your-contributors. This code merely got the contributors as defined by GitHub, most likely due to time constraints. But what you outline above is more inclusive, and therefore, IMHO, better. 👍

[RichardLitt]

https://github.com/RichardLitt/name-your-contributors works now. Might be worth looking into and using for this.

[daviddias]

@ipfs/dx team, Can I ask for your help to get this get-gh-contributors running so that we get the latest head count?

[victorb]

@diasdavid seems name-your-contributors (by RichardLitt) mentioned above would do the same thing as the program whyrusleeping suggested, could we just use that? What's the ask here, to have it running with API endpoints rather than as a CLI?

[daviddias]

@victorbjelkholm I need the dataset -- https://github.com/ipfs/contributors-hex-grid/blob/master/public-vertical/js/contributors.js -- to be up to date.

I'm fine with either tool, as long as it is accurate.

[RichardLitt]

Well, name-your-contributors is to my knowledge accurate. It should work for this. @victorbjelkholm let me know if you need help.

[daviddias]

Note to self. The go program expects a data folder to exist otherwise it panics.

[daviddias]

@RichardLitt name-your-contributors is looking rad! It is just unfortunate that it currently fails for orgs mntnr/name-your-contributors#72

[daviddias]

Just finished running the script and learned that something has changed because I'm no longer getting 3000 +++ like last time, just 2441 for this round. Trying to find the time to figure this one out.

Update: found the issue, it was just picking up on 10 repos from each org. Increased that now. Need to crawl the rest.

[daviddias]

New updated number 3758

[RichardLitt]

@daviddias Sorry that it fails! Did you get it working? I've been slammed the past couple of days.