Merge pull request #1080 from libp2p/feat/release-process

doc: add a basic release process

.github/ISSUE_TEMPLATE/release.md

@@ -0,0 +1,54 @@
+---
+name: 'Libp2p Release'
+about: 'Start a new libp2p release.'
+---
+
+## 🗺 What's left for release
+
+<List of items with remaining PRs and/or Issues to be considered for this release>
+
+## 🔦 Highlights
+
+< top highlights for this release notes >
+
+## Changelog
+
+< changelog generated by scripts/mkreleaselog >
+
+## ✅ Release Checklist
+
+- [ ] **Stage 0 - Finishing Touches**
+    - [ ] Go through relevant libp2p repos looking for unreleased changes that should make it into the release. If you find any, cut releases.
+    - [ ] Run `go get -u ./...` to see if there are any out-of-date deps that look important. If there are, bubble them. Try to avoid _directly_ updating indirect deps in go-libp2p's go.mod when possible.
+    - [ ] Fork a new `release-vX.Y.Z` branch from `master`, freezing master.
+    - [ ] Make sure local tests are passing.
+- [ ] **Stage 1 - Upstream Testing**
+  - Create testing branches in lotus & go-ipfs with the new go-libp2p release and run CI/tests. Many upstream projects are tested in CI, but lotus & go-ipfs are not.
+    - [ ] [filecoin-project/lotus](https://github.com/filecoin-project/lotus)
+    - [ ] [ipfs/go-ipfs](https://github.com/ipfs/go-ipfs)
+  - _(someday)_ Run upstream testground tests. Unfortunately, this is too time consuming at the moment.
+    - _(someday)_ Run bitswap testground tests.
+    - _(someday)_ Run DHT testground tests.
+- [ ] **Stage 2 - Infrastructure Testing**
+  - How: Using the testing branches created above, work with the infrastructure team to deploy the new libp2p versions to our infrastructure.
+  - Where:
+    - [ ] A go-ipfs gateway.
+    - [ ] A go-ipfs bootstrapper.
+    - [ ] A go-ipfs preload node.
+    - [ ] A hydra booster.
+    - [ ] A Filecoin bootstrap node.
+    - [ ] Calibration net
+  - What: Look at pprof profile dumps, especially CPU profiles and heap allocation profiles, noting any significant changes. Make sure nothing crashes.
+- [ ] **Stage 3 - Release**
+  - [ ] Merge the release PR.
+  - [ ] Tag the release on master.
+  - [ ] Publish the release through the GitHub UI, adding the release notes. Some users rely on this to receive notifications of new releases.
+  - [ ] Announce the release on the [discuss.libp2p.io](https://discuss.libp2p.io).
+- [ ] **Stage 4 - Update Upstream**
+  - [ ] Update the upstream testing branches to the final release and create PRs.
+    - [ ] [filecoin-project/lotus](https://github.com/filecoin-project/lotus)
+    - [ ] [ipfs/go-bitswap](https://github.com/ipfs/go-bitswap)
+    - [ ] [ipfs/go-ipfs](https://github.com/ipfs/go-ipfs)
+    - [ ] [libp2p/go-libp2p-kad-dht](https://github.com/libp2p/go-libp2p-kad-dht)
+    - [ ] [libp2p/go-libp2p-pubsub](https://github.com/libp2p/go-libp2p-pubsub)
+    - [ ] [libp2p/go-libp2p-daemon](https://github.com/libp2p/go-libp2p-daemon)

README.md

@@ -110,6 +110,10 @@ If you experience any issues migrating from gx to gomod, please [join the discus
 
 `go test ./...` will run all tests in the repo.
 
+### Releasing
+
+Please start a release by opening a new [Libp2p Release](https://github.com/libp2p/go-libp2p/issues/new?assignees=&labels=kind/tracking&template=release.md) issue.
+
 ### Packages
 
 > This table is generated using the module [`package-table`](https://github.com/ipfs-shipyard/package-table) with `package-table --data=package-list.json`.

scripts/mkreleaselog

@@ -0,0 +1,244 @@
+#!/bin/zsh
+#set -x
+set -euo pipefail
+export GO111MODULE=on
+export GOPATH="$(go env GOPATH)"
+
+alias jq="jq --unbuffered"
+
+AUTHORS=(
+    # orgs
+    ipfs
+    ipld
+    libp2p
+    multiformats
+    filecoin-project
+    ipfs-shipyard
+
+    # Authors of personal repos used by go-libp2p that should be mentioned in the
+    # release notes.
+    whyrusleeping
+    Kubuxu
+    jbenet
+    Stebalien
+    marten-seemann
+    hsanjuan
+    lucas-clemente
+    warpfork
+)
+
+[[ -n "${REPO_FILTER+x}" ]] || REPO_FILTER="github.com/(${$(printf "|%s" "${AUTHORS[@]}"):1})"
+
+[[ -n "${IGNORED_FILES+x}" ]] || IGNORED_FILES='^\(\.gx\|package\.json\|\.travis\.yml\|go.mod\|go\.sum|\.github|\.circleci\)$'
+
+NL=$'\n'
+
+ROOT_DIR="$(git rev-parse --show-toplevel)"
+
+msg() {
+    echo "$*" >&2
+}
+
+statlog() {
+    local module="$1"
+    local rpath="$GOPATH/src/$(strip_version "$module")"
+    local start="${2:-}"
+    local end="${3:-HEAD}"
+    local mailmap_file="$rpath/.mailmap"
+    if ! [[ -e "$mailmap_file" ]]; then
+        mailmap_file="$ROOT_DIR/.mailmap"
+    fi
+
+    git -C "$rpath" -c mailmap.file="$mailmap_file" log --use-mailmap --shortstat --no-merges --pretty="tformat:%H%n%aN%n%aE" "$start..$end" | while 
+        read hash
+        read name
+        read email
+        read _ # empty line
+        read changes
+    do
+        changed=0
+        insertions=0
+        deletions=0
+        while read count event; do
+            if [[ "$event" =~ ^file ]]; then
+                changed=$count
+            elif [[ "$event" =~ ^insertion ]]; then
+                insertions=$count
+            elif [[ "$event" =~ ^deletion ]]; then
+                deletions=$count
+            else
+                echo "unknown event $event" >&2
+                exit 1
+            fi
+        done<<<"${changes//,/$NL}"
+
+        jq -n \
+           --arg "hash" "$hash" \
+           --arg "name" "$name" \
+           --arg "email" "$email" \
+           --argjson "changed" "$changed" \
+           --argjson "insertions" "$insertions" \
+           --argjson "deletions" "$deletions" \
+           '{Commit: $hash, Author: $name, Email: $email, Files: $changed, Insertions: $insertions, Deletions: $deletions}'
+    done
+}
+
+# Returns a stream of deps changed between $1 and $2.
+dep_changes() {
+    {
+        <"$1"
+        <"$2"
+    } | jq -s 'JOIN(INDEX(.[0][]; .Path); .[1][]; .Path; {Path: .[0].Path, Old: (.[1] | del(.Path)), New: (.[0] | del(.Path))}) | select(.New.Version != .Old.Version)'
+}
+
+# resolve_commits resolves a git ref for each version.
+resolve_commits() {
+    jq '. + {Ref: (.Version|capture("^((?<ref1>.*)\\+incompatible|v.*-(0\\.)?[0-9]{14}-(?<ref2>[a-f0-9]{12})|(?<ref3>v.*))$") | .ref1 // .ref2 // .ref3)}'
+}
+
+pr_link() {
+    local repo="$1"
+    local prnum="$2"
+    local ghname="${repo##github.com/}"
+    printf -- "[%s#%s](https://%s/pull/%s)" "$ghname" "$prnum" "$repo" "$prnum"
+}
+
+# Generate a release log for a range of commits in a single repo.
+release_log() {
+    setopt local_options BASH_REMATCH
+
+    local module="$1"
+    local start="$2"
+    local end="${3:-HEAD}"
+    local repo="$(strip_version "$1")"
+    local dir="$GOPATH/src/$repo"
+
+    local commit pr
+    git -C "$dir" log \
+        --format='tformat:%H %s' \
+        --first-parent \
+        "$start..$end" |
+        while read commit subject; do
+            # Skip gx-only PRs.
+            git -C "$dir" diff-tree --no-commit-id --name-only "$commit^" "$commit" |
+                grep -v "${IGNORED_FILES}" >/dev/null || continue
+
+            if [[ "$subject" =~ '^Merge pull request #([0-9]+) from' ]]; then
+                local prnum="${BASH_REMATCH[2]}"
+                local desc="$(git -C "$dir" show --summary --format='tformat:%b' "$commit" | head -1)"
+                printf -- "- %s (%s)\n" "$desc" "$(pr_link "$repo" "$prnum")"
+            elif [[ "$subject" =~ '\(#([0-9]+)\)$' ]]; then
+                local prnum="${BASH_REMATCH[2]}"
+                printf -- "- %s (%s)\n" "$subject" "$(pr_link "$repo" "$prnum")"
+            else
+                printf -- "- %s\n" "$subject"
+            fi
+        done
+}
+
+indent() {
+    sed -e 's/^/  /'
+}
+
+mod_deps() {
+    go list -mod=mod -json -m all | jq 'select(.Version != null)'
+}
+
+ensure() {
+    local repo="$(strip_version "$1")"
+    local commit="$2"
+    local rpath="$GOPATH/src/$repo"
+    if [[ ! -d "$rpath" ]]; then
+        msg "Cloning $repo..."
+        git clone "http://$repo" "$rpath" >&2
+    fi
+
+    if ! git -C "$rpath" rev-parse --verify "$commit" >/dev/null; then
+        msg "Fetching $repo..."
+        git -C "$rpath" fetch --all >&2
+    fi
+
+    git -C "$rpath" rev-parse --verify "$commit" >/dev/null || return 1
+}
+
+statsummary() {
+    jq -s 'group_by(.Author)[] | {Author: .[0].Author, Commits: (. | length), Insertions: (map(.Insertions) | add), Deletions: (map(.Deletions) | add), Files: (map(.Files) | add)}' |
+        jq '. + {Lines: (.Deletions + .Insertions)}'
+}
+
+strip_version() {
+    local repo="$1"
+    if [[ "$repo" =~ '.*/v[0-9]+$' ]]; then
+        repo="$(dirname "$repo")"
+    fi
+    echo "$repo"
+}
+
+recursive_release_log() {
+    local start="${1:-$(git tag -l | sort -V | grep -v -- '-rc' | grep 'v'| tail -n1)}"
+    local end="${2:-$(git rev-parse HEAD)}"
+    local repo_root="$(git rev-parse --show-toplevel)"
+    local module="$(go list -m)"
+    local dir="$(go list -m -f '{{.Dir}}')"
+
+    if [[ "${GOPATH}/${module}" -ef "${dir}" ]]; then
+        echo "This script requires the target module and all dependencies to live in a GOPATH."
+        return 1
+    fi
+
+    (
+        local result=0
+        local workspace="$(mktemp -d)"
+        trap "$(printf 'rm -rf "%q"' "$workspace")" INT TERM EXIT
+        cd "$workspace"
+
+        echo "Computing old deps..." >&2
+        git -C "$repo_root" show "$start:go.mod" >go.mod
+        mod_deps | resolve_commits | jq -s > old_deps.json
+
+        echo "Computing new deps..." >&2
+        git -C "$repo_root" show "$end:go.mod" >go.mod
+        mod_deps | resolve_commits | jq -s > new_deps.json
+
+        rm -f go.mod go.sum
+
+        printf -- "Generating Changelog for %s %s..%s\n" "$module" "$start" "$end" >&2
+
+        printf -- "- %s:\n" "$module"
+        release_log "$module" "$start" "$end" | indent
+
+
+        statlog "$module" "$start" "$end" > statlog.json
+
+        dep_changes old_deps.json new_deps.json |
+            jq --arg filter "$REPO_FILTER" 'select(.Path | match($filter))' |
+            # Compute changelogs
+            jq -r '"\(.Path) \(.New.Version) \(.New.Ref) \(.Old.Version) \(.Old.Ref // "")"' |
+            while read module new new_ref old old_ref; do
+                if ! ensure "$module" "$new_ref"; then
+                    result=1
+                    local changelog="failed to fetch repo"
+                else
+                    statlog "$module" "$old_ref" "$new_ref" >> statlog.json
+                    local changelog="$(release_log "$module" "$old_ref" "$new_ref")"
+                fi
+                if [[ -n "$changelog" ]]; then
+                    printf -- "- %s (%s -> %s):\n" "$module" "$old" "$new"
+                    echo "$changelog" | indent
+                fi
+            done
+
+        echo
+        echo "Contributors"
+        echo
+
+        echo "| Contributor | Commits | Lines ± | Files Changed |"
+        echo "|-------------|---------|---------|---------------|"
+        statsummary <statlog.json |
+            jq -s 'sort_by(.Lines) | reverse | .[]' |
+            jq -r '"| \(.Author) | \(.Commits) | +\(.Insertions)/-\(.Deletions) | \(.Files) |"'
+        return "$status"
+    )
+}
+
+recursive_release_log "$@"