Skip to content

Commit

Permalink
Add istanbul package e2e_test coverage job to CI (#1692)
Browse files Browse the repository at this point in the history
Add tool to output package level stats from coverage files

Usage:

go run tools/parsecov/main.go coverage.out

Add coverage job for CI that posts report of coverage
generated by the e2e tests across the istanbul package on PRs.
  • Loading branch information
piersy authored Sep 21, 2021
1 parent 46dac57 commit 26cba2a
Show file tree
Hide file tree
Showing 2 changed files with 173 additions and 0 deletions.
48 changes: 48 additions & 0 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,50 @@ jobs:
- run: go run build/ci.go test -coverage
- run: bash <(curl -s https://codecov.io/bash)

istanbul-e2e-coverage:
executor: golang
resource_class: medium+
steps:
- attach_workspace:
at: ~/repos
- restore_cache:
keys:
- go-mod-v1-{{ checksum "go.sum" }}
# Run the tests with coverage parse the coverage and output the summary
- run:
name: Run tests and print coverage summary
command: |
go test -coverprofile cov.out -coverpkg ./consensus/istanbul/... ./e2e_test
go run tools/parsecov/main.go -packagePrefix github.com/celo-org/celo-blockchain/ cov.out > summary
cat summary
- run:
name: Post summary comment on PR
command: |
# Only post on PR if this build is running on a PR. If this build
# is running on a PR then CIRCLE_PULL_REQUEST contains the link to
# the PR.
if [[ ! -z ${CIRCLE_PULL_REQUEST} ]] ; then
# Build comment
echo "Coverage from tests in \`./e2e_test/...\` for \`./consensus/istanbul/...\` at commit ${CIRCLE_SHA1}" > comment
echo "\`\`\`" >> comment
cat summary >> comment
echo "\`\`\`" >> comment
# This command is quite involved, its posting the comment on the
# associated PR.
# ${CIRCLE_PULL_REQUEST##*/} expands to just the pr number at the
# end of the PR link.
# "{\"body\":\"`awk -v ORS='\\\\n' '1' comment`\"}" evaluates to
# a json object with comment as the content of body with newlines
# replaced by '\n'. Using backtics causes there to be a round of
# backslash processing on the command before execution, so we
# need to double the backslashes in the awk command.
curl -u piersy:${PR_COMMENT_TOKEN} -X POST \
https://api.github.com/repos/celo-org/celo-blockchain/issues/${CIRCLE_PULL_REQUEST##*/}/comments \
-d "{\"body\":\"`awk -v ORS='\\\\n' '1' comment`\"}" ;
fi
lint:
executor: golang
steps:
Expand Down Expand Up @@ -338,6 +382,10 @@ workflows:
requires:
- build-geth
- prepare-system-contracts
- istanbul-e2e-coverage:
requires:
- build-geth
- prepare-system-contracts
- android
- ios
- publish-mobile-client:
Expand Down
125 changes: 125 additions & 0 deletions tools/parsecov/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
package main

import (
"bufio"
"fmt"
"log"
"os"
"path"
"sort"
"strconv"
"strings"

"gopkg.in/urfave/cli.v1"
)

var (
packagePrefixFlagName = "packagePrefix"
)

type count struct {
totalStatements int
coveredStatements int
}

func (c count) percentage() float32 {
return 100 * float32(c.coveredStatements) / float32(c.totalStatements)
}

func parse(c *cli.Context) error {
packageCoverage := make(map[string]count)
profile, err := os.Open(c.Args()[0])
if err != nil {
return err
}
defer profile.Close()

// First line is "mode: foo", where foo is "set", "count", or "atomic".
// Rest of file is in the format
// encoding/base64/base64.go:34.44,37.40 3 1
// where the fields are: name.go:line.column,line.column numberOfStatements count

// This program only handles set mode
s := bufio.NewScanner(profile)
s.Scan()
line := s.Text()
if line != "mode: set" {
return fmt.Errorf("invalid coverage mode, expecting 'mode: set' as the first line, but got %q", line)
}
linenum := 1
for s.Scan() {
line := s.Text()
linenum++
lastColon := strings.LastIndex(line, ":")
if lastColon == -1 {
return fmt.Errorf("line %d invalid: %q", linenum, line)
}

packageName := path.Dir(line[:lastColon])
numStatements, covered, err := getStats(line[lastColon+1:])
if err != nil {
return fmt.Errorf("line %d invalid: %q", linenum, line)
}
c := packageCoverage[packageName]
c.totalStatements += numStatements
if covered {
c.coveredStatements += numStatements
}
packageCoverage[packageName] = c
}

packageNames := make([]string, 0, len(packageCoverage))
for name := range packageCoverage {
packageNames = append(packageNames, name)
}
sort.Strings(packageNames)

var totalCount count
for _, v := range packageCoverage {
totalCount.totalStatements += v.totalStatements
totalCount.coveredStatements += v.coveredStatements
}
fmt.Printf("coverage: %5.1f%% of statements across all listed packages\n", totalCount.percentage())

for _, name := range packageNames {
cov := packageCoverage[name].percentage()
fmt.Printf("coverage: %5.1f%% of statements in %s\n", cov, strings.TrimPrefix(name, c.String(packagePrefixFlagName)))
}

return nil
}

// Gets the stats from the end of the line, how many statements and were they
// covered?
//
// E.G line -> encoding/base64/base64.go:34.44,37.40 3 1
// End of line -> 34.44,37.40 3 1
// Would return 3 true
func getStats(lineEnd string) (numStatements int, covered bool, err error) {
parts := strings.Split(lineEnd, " ")
if len(parts) != 3 {
return 0, false, fmt.Errorf("invalid line end")
}
numStatements, err = strconv.Atoi(parts[1])
return numStatements, parts[2] == "1", err
}

func main() {
app := cli.NewApp()
app.Name = "parsecov"
app.Usage = `parses coverage files outputting a per package breakdown
of coverage as well as a total for all the packages.`
app.Flags = []cli.Flag{
cli.StringFlag{
Name: packagePrefixFlagName,
Usage: `a common prefix that is stripped from the front
of all packages in order to make output more concise.`,
},
}
app.Action = parse

err := app.Run(os.Args)
if err != nil {
log.Fatalf("Failed to parse coverage: %v", err)
}
}

0 comments on commit 26cba2a

Please sign in to comment.