feat: extract multiple metrics from golang benchmarks (#177)

* use a more recent version of go * allows go benchmarks to work with extra metrics Co-authored-by: Joe Kimmel <[email protected]>
benchmark-action · Jul 7, 2023 · 0c70f15 · 0c70f15
1 parent 3dc9c9f
commit 0c70f15
Show file tree

Hide file tree

Showing 11 changed files with 95 additions and 26 deletions.
diff --git a/.github/workflows/ci-results-repo.yml b/.github/workflows/ci-results-repo.yml
@@ -57,7 +57,9 @@ jobs:
       - uses: actions/setup-node@v3
         with:
           node-version: 16
-      - uses: actions/setup-go@v1
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "stable"
       - uses: actions/cache@v1
         with:
           path: ~/.npm
@@ -336,7 +338,9 @@ jobs:
           key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }}
       - run: npm ci
       - run: npm run build
-      - uses: actions/setup-go@v1
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "stable"
       - name: Run benchmark
         run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt
       - name: Download previous benchmark data

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -49,7 +49,9 @@ jobs:
       - uses: actions/setup-node@v3
         with:
           node-version: 16
-      - uses: actions/setup-go@v1
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "stable"
       - uses: actions/cache@v1
         with:
           path: ~/.npm
@@ -346,7 +348,9 @@ jobs:
           key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }}
       - run: npm ci
       - run: npm run build
-      - uses: actions/setup-go@v1
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "stable"
       - name: Run benchmark
         run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt
       - name: Download previous benchmark data

diff --git a/.github/workflows/commit-comment.yml b/.github/workflows/commit-comment.yml
@@ -14,7 +14,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-go@v1
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "stable"
       - name: Run benchmark
         run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt
       - name: Download previous benchmark data

diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
@@ -14,7 +14,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-go@v1
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "stable"
       - name: Run benchmark
         run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt
 

diff --git a/.github/workflows/minimal.yml b/.github/workflows/minimal.yml
@@ -14,7 +14,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-go@v1
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "stable"
       - name: Run benchmark
         run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt
       - name: Download previous benchmark data

diff --git a/README.md b/README.md
@@ -151,7 +151,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-go@v1
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "stable"
       # Run benchmark with `go test -bench` and stores the output to a file
       - name: Run benchmark
         run: go test -bench 'BenchmarkFib' | tee output.txt
@@ -285,7 +287,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-go@v1
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "stable"
       # Run benchmark with `go test -bench` and stores the output to a file
       - name: Run benchmark
         run: go test -bench 'BenchmarkFib' | tee output.txt

diff --git a/examples/go/fib_test.go b/examples/go/fib_test.go
@@ -15,3 +15,10 @@ func BenchmarkFib20(b *testing.B) {
 		var _ = Fib(20)
 	}
 }
+
+func BenchmarkFib20WithAuxMetric(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		var _ = Fib(20)
+	}
+	b.ReportMetric(4.0, "auxMetricUnits")
+}
diff --git a/examples/go/go.mod b/examples/go/go.mod
@@ -0,0 +1,3 @@
+module github.com/github-action-benchmark/github-action-benchmark/examples/go
+
+go 1.20
diff --git a/src/extract.ts b/src/extract.ts
@@ -345,26 +345,40 @@ function extractGoResult(output: string): BenchmarkResult[] {
     // Example:
     //   BenchmarkFib20-8           30000             41653 ns/op
     //   BenchmarkDoWithConfigurer1-8            30000000                42.3 ns/op
-    const reExtract = /^(Benchmark\w+(?:\/?[\w()$%^&*-=,]*?)*?)(-\d+)?\s+(\d+)\s+([0-9.]+)\s+(.+)$/;
 
-    for (const line of lines) {
-        const m = line.match(reExtract);
-        if (m === null) {
-            continue;
-        }
+    // Example if someone has used the ReportMetric function to add additional metrics to each benchmark:
+    // BenchmarkThing-16    	       1	95258906556 ns/op	        64.02 UnitsForMeasure2	        31.13 UnitsForMeasure3
 
-        const name = m[1];
-        const procs = m[2] !== undefined ? m[2].slice(1) : null;
-        const times = m[3];
-        const value = parseFloat(m[4]);
-        const unit = m[5];
+    // reference, "Proposal: Go Benchmark Data Format": https://go.googlesource.com/proposal/+/master/design/14313-benchmark-format.md
+    // "A benchmark result line has the general form: <name> <iterations> <value> <unit> [<value> <unit>...]"
+    // "The fields are separated by runs of space characters (as defined by unicode.IsSpace), so the line can be parsed with strings.Fields. The line must have an even number of fields, and at least four."
+    const reExtract =
+        /^(?<name>Benchmark\w+(?:\/?[\w()$%^&*-=,]*?)*?)(?<procs>-\d+)?\s+(?<times>\d+)\s+(?<remainder>.+)$/;
 
-        let extra = `${times} times`;
-        if (procs !== null) {
-            extra += `\n${procs} procs`;
+    for (const line of lines) {
+        const m = line.match(reExtract);
+        if (m?.groups) {
+            const procs = m.groups.procs !== undefined ? m.groups.procs.slice(1) : null;
+            const times = m.groups.times;
+            const remainder = m.groups.remainder;
+
+            const pieces = remainder.split(/[ \t]+/);
+            for (let i = 0; i < pieces.length; i = i + 2) {
+                let extra = `${times} times`.replace(/\s\s+/g, ' ');
+                if (procs !== null) {
+                    extra += `\n${procs} procs`;
+                }
+                const value = parseFloat(pieces[i]);
+                const unit = pieces[i + 1];
+                let name;
+                if (pieces.length > 2) {
+                    name = m.groups.name + ' - ' + unit;
+                } else {
+                    name = m.groups.name;
+                }
+                ret.push({ name, value, unit, extra });
+            }
         }
-
-        ret.push({ name, value, unit, extra });
     }
 
     return ret;

diff --git a/test/data/extract/go_output.txt b/test/data/extract/go_output.txt
@@ -6,5 +6,8 @@ BenchmarkFib/my_tabled_benchmark_-_10-8    					5000000		325 ns/op
 BenchmarkFib/my_tabled_benchmark_-_20						30000		40537.123 ns/op
 BenchmarkFib/my/tabled/benchmark_-_20						30001		40537.456 ns/op
 BenchmarkFib/my/tabled/benchmark_-_20,var1=13,var2=14		30001		40537.456 ns/op
+BenchmarkFib11-16    	                                    4587789	    262.7 ns/op	         3.000 auxMetricUnits
+BenchmarkFib22-16    	                                    37653	    31915 ns/op	         4.000 auxMetricUnits
+PASS
 PASS
 ok  	_/Users/rhayasd/Develop/github.com/benchmark-action/github-action-benchmark/examples/go	3.614s
diff --git a/test/extract.spec.ts b/test/extract.spec.ts
@@ -209,6 +209,30 @@ describe('extractResult()', function () {
                     value: 40537.456,
                     extra: '30001 times',
                 },
+                {
+                    name: 'BenchmarkFib11 - ns/op',
+                    unit: 'ns/op',
+                    value: 262.7,
+                    extra: '4587789 times\n16 procs',
+                },
+                {
+                    name: 'BenchmarkFib11 - auxMetricUnits',
+                    unit: 'auxMetricUnits',
+                    value: 3,
+                    extra: '4587789 times\n16 procs',
+                },
+                {
+                    name: 'BenchmarkFib22 - ns/op',
+                    unit: 'ns/op',
+                    value: 31915,
+                    extra: '37653 times\n16 procs',
+                },
+                {
+                    name: 'BenchmarkFib22 - auxMetricUnits',
+                    unit: 'auxMetricUnits',
+                    value: 4,
+                    extra: '37653 times\n16 procs',
+                },
             ],
         },
         {
@@ -466,7 +490,7 @@ describe('extractResult()', function () {
             A.equal(bench.commit, dummyWebhookPayload.head_commit);
             A.ok(bench.date <= Date.now(), bench.date.toString());
             A.equal(bench.tool, test.tool);
-            A.deepEqual(test.expected, bench.benches);
+            A.deepEqual(bench.benches, test.expected);
         });
     }