Skip to content

Commit

Permalink
add junit report + buildkite upload (#68)
Browse files Browse the repository at this point in the history
Fixes #65

and re-arranged some stuff:
* split out all report renders to their own file
* update result struct to track per-test runtime
* disable failing web5-kt test
* use web5-js test from web5-js repo, drop web5-js test from this repo
* renamed the go package to reflect the current repo name
finn-block authored Nov 28, 2023

Verified

This commit was signed with the committer’s verified signature.
fiji-flo Florian Dieminger
1 parent 5385c51 commit df9d113
Showing 29 changed files with 349 additions and 2,990 deletions.
6 changes: 5 additions & 1 deletion .github/workflows/pages.yaml
Original file line number Diff line number Diff line change
@@ -11,7 +11,11 @@ jobs:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
- name: run for all SDKs
run: cd test-harness && go run ./cmd/web5-test-harness many sdks/* && mv _site ../_site
run: |
git clone https://github.com/TBD54566975/web5-js.git
git clone https://github.com/TBD54566975/web5-kt.git
cd test-harness
go run ./cmd/web5-test-harness many ../web5-* && mv _site ../_site
- name: Copy test-vectors
run: cp -r ./web5-test-vectors _site/
- uses: actions/upload-pages-artifact@v2
8 changes: 6 additions & 2 deletions .github/workflows/self-test.yaml
Original file line number Diff line number Diff line change
@@ -10,11 +10,15 @@ jobs:
matrix:
SDK:
- web5-js
- web5-kt
# - web5-kt
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
- name: self test
run: cd test-harness && go run ./cmd/web5-test-harness one sdks/$SDK
run: |
git clone https://github.com/TBD54566975/${SDK} sdk
cd test-harness
go run ./cmd/web5-test-harness one ../sdk
env:
SDK: ${{ matrix.SDK }}
BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_ANALYTICS_TOKEN }}
6 changes: 3 additions & 3 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
@@ -5,7 +5,7 @@ There are many ways to be an open source contributor, and we're here to help you

* Propose ideas in our
[discord](https://discord.gg/tbd)
* Raise an issue or feature request in our [issue tracker](https://github.com/TBD54566975/web5-spec/issues)
* Raise an issue or feature request in our [issue tracker](https://github.com/TBD54566975/sdk-development/issues)
* Help another contributor with one of their questions, or a code review
* Suggest improvements to our Getting Started documentation by supplying a Pull Request
* Evangelize our work together in conferences, podcasts, and social media spaces.
@@ -171,7 +171,7 @@ $> ./gradlew clean build test
### Issues

Anyone from the community is welcome (and encouraged!) to raise issues via
[GitHub Issues](https://github.com/TBD54566975/web5-spec/issues).
[GitHub Issues](https://github.com/TBD54566975/sdk-development/issues).

### Discussions

@@ -181,7 +181,7 @@ We advocate an asynchronous, written debate model - so write up your thoughts an

### Continuous Integration

Build and Test cycles are run on every commit to every branch on [GitHub Actions](https://github.com/TBD54566975/web5-spec/actions).
Build and Test cycles are run on every commit to every branch on [GitHub Actions](https://github.com/TBD54566975/sdk-development/actions).

## Contribution

2 changes: 1 addition & 1 deletion test-harness/cmd/web5-test-harness/test-many.go
Original file line number Diff line number Diff line change
@@ -4,7 +4,7 @@ import (
"fmt"
"os"

"github.com/TBD54566975/web5-spec/reports"
"github.com/TBD54566975/sdk-development/reports"
"github.com/spf13/cobra"
"golang.org/x/exp/slog"
)
110 changes: 107 additions & 3 deletions test-harness/cmd/web5-test-harness/test-one.go
Original file line number Diff line number Diff line change
@@ -1,15 +1,20 @@
package main

import (
"bytes"
"context"
"fmt"
"io"
"mime/multipart"
"net/http"
"os"
"path/filepath"
"strings"
"time"

"github.com/TBD54566975/web5-spec/openapi"
"github.com/TBD54566975/web5-spec/reports"
"github.com/TBD54566975/web5-spec/tests"
"github.com/TBD54566975/sdk-development/openapi"
"github.com/TBD54566975/sdk-development/reports"
"github.com/TBD54566975/sdk-development/tests"
"github.com/spf13/cobra"
"golang.org/x/exp/slog"
)
@@ -43,6 +48,20 @@ var (
}
}

buildkiteToken := os.Getenv("BUILDKITE_ANALYTICS_TOKEN")
if buildkiteToken != "" {
if err := buildkiteUpload(report, buildkiteToken); err != nil {
slog.Error("error sending results to buildkite", "error", err)
}
}

junitFile := os.Getenv("JUNIT_REPORT")
if junitFile != "" {
if err := reports.WriteJunitToFile(report, junitFile); err != nil {
slog.Error("error writing junit report", "file", junitFile, "error", err)
}
}

if !report.IsPassing() {
os.Exit(1)
}
@@ -129,3 +148,88 @@ func testOne(dir string) (reports.Report, error) {
func init() {
root.AddCommand(testOneCmd)
}

func buildkiteUpload(report reports.Report, buildkiteToken string) error {
slog.Info("uploading junit report to buildkit")

var body bytes.Buffer
writer := multipart.NewWriter(&body)

formFields := map[string]string{
"format": "junit",
"run_env[CI]": "github_actions",
"run_env[key]": fmt.Sprintf("%s-%s-%s", os.Getenv("GITHUB_ACTION"), os.Getenv("GITHUB_RUN_NUMBER"), os.Getenv("GITHUB_RUN_ATTEMPT")),
"run_env[number]": os.Getenv("GITHUB_RUN_NUMBER"),
"run_env[branch]": os.Getenv("GITHUB_REF"),
"run_env[commit_sha]": os.Getenv("GITHUB_SHA"),
"run_env[url]": fmt.Sprintf("https://github.com/%s/actions/runs/%s", os.Getenv("GITHUB_REPOSITORY"), os.Getenv("GITHUB_RUN_ID")),
}
for k, v := range formFields {
if err := addFormValue(writer, k, v); err != nil {
return err
}
}

part, err := writer.CreateFormFile("data", "web5-test-harness.xml")
if err != nil {
slog.Error("error creating form file")
return err
}

if err := reports.WriteJunit(report, part); err != nil {
slog.Error("error generating junit report")
return err
}

if err := writer.Close(); err != nil {
slog.Error("error closing multi-part form writer")
return err
}

req, err := http.NewRequest(http.MethodPost, "https://analytics-api.buildkite.com/v1/uploads", &body)
if err != nil {
slog.Error("error constructing request to buildkite")
return err
}

req.Header.Add("Content-Type", writer.FormDataContentType())
req.Header.Add("Authorization", fmt.Sprintf("Token token=%s", buildkiteToken))

resp, err := http.DefaultClient.Do(req)
if err != nil {
slog.Error("error uploading results to buildkite")
return err
}
defer resp.Body.Close()

responseBody := bytes.Buffer{}
if _, err := io.Copy(&responseBody, resp.Body); err != nil {
slog.Error("error reading response from buildkite")
return err
}

if resp.StatusCode > 299 {
slog.Error("unexpected response status from buildkite", "status", resp.Status, "response", body.String())
return fmt.Errorf("unexpected %s", resp.Status)
}

slog.Info("successfully uploaded results to buildkite", "response", body.String())

return nil
}

func addFormValue(writer *multipart.Writer, key, value string) error {
field, err := writer.CreateFormField(key)
if err != nil {
slog.Error("error creating form field", "key", key, "value", value)
return err
}

_, err = io.Copy(field, strings.NewReader(value))
if err != nil {
slog.Error("error writing form field value", "key", key, "value", value)
return err
}

return nil
}
2 changes: 1 addition & 1 deletion test-harness/go.mod
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
module github.com/TBD54566975/web5-spec
module github.com/TBD54566975/sdk-development

go 1.20

62 changes: 62 additions & 0 deletions test-harness/reports/html.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
package reports

import (
"fmt"
"os"
"strings"

"golang.org/x/exp/slog"
)

func sanatizeHTML(dirty error) string {
clean := strings.ReplaceAll(dirty.Error(), "<", "&lt;")
clean = strings.ReplaceAll(clean, ">", "&gt;")
clean = strings.ReplaceAll(clean, "\n", "\\\\n")

return clean
}

type htmlTemplateInput struct {
Reports []Report
Tests map[string][]string
}

func WriteHTML(reports []Report, filename string) error {
slog.Info("writing html report")

testmap := make(map[string]map[string]bool)
for _, report := range reports {
for category, tests := range report.Results {
if _, ok := tests[category]; !ok {
testmap[category] = map[string]bool{}
}

for test := range tests {
testmap[category][test] = true
}
}
}

templateInput := htmlTemplateInput{
Reports: reports,
Tests: make(map[string][]string),
}

for category, tests := range testmap {
for test := range tests {
templateInput.Tests[category] = append(templateInput.Tests[category], test)
}
}

f, err := os.Create(filename)
if err != nil {
return fmt.Errorf("error opening %s: %v", filename, err)
}
defer f.Close()

if err := htmlTemplates.ExecuteTemplate(f, "report-template.html", templateInput); err != nil {
return err
}

return nil
}
55 changes: 55 additions & 0 deletions test-harness/reports/junit.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
package reports

import (
"fmt"
"io"
"os"
"time"
)

func durationToJunit(t time.Duration) string {
return fmt.Sprintf("%f", float64(t)/float64(time.Second))
}

type junitTemplateInput struct {
TimeTotal time.Duration
TimePerTestSuite map[string]time.Duration
Report Report
}

func WriteJunitToFile(report Report, filename string) error {
f, err := os.Create(filename)
if err != nil {
return err
}
defer f.Close()

if err := WriteJunit(report, f); err != nil {
return err
}

return nil
}

func WriteJunit(report Report, writer io.Writer) error {
templateInput := junitTemplateInput{
Report: report,
TimePerTestSuite: map[string]time.Duration{},
}

for testsuite, results := range report.Results {
var testsuiteTime time.Duration
for _, result := range results {
testsuiteTime = testsuiteTime + result.Time
templateInput.TimeTotal = templateInput.TimeTotal + result.Time
}

templateInput.TimePerTestSuite[testsuite] = testsuiteTime
}

if err := textTemplates.ExecuteTemplate(writer, "report-template.junit.xml", templateInput); err != nil {
return err
}

return nil
}
18 changes: 18 additions & 0 deletions test-harness/reports/md.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
package reports

import "os"

func WriteMarkdown(report Report, filename string) error {
f, err := os.Create(filename)
if err != nil {
return err
}
defer f.Close()

err = textTemplates.ExecuteTemplate(f, "report-template.md", report)
if err != nil {
return err
}

return nil
}
6 changes: 3 additions & 3 deletions test-harness/reports/report-template.html
Original file line number Diff line number Diff line change
@@ -28,12 +28,12 @@ <h2>{{ $category }}</h2>
{{ range $.Reports }}
<td>
{{ if not (index (index .Results $category) $test) }}
{{ index (index .Results $category) $test | getEmoji }}
{{ (index (index .Results $category) $test).GetEmoji }}
{{ else }}
<details>
<summary>{{ index (index .Results $category) $test | getEmoji }}</summary>
<summary>{{ (index (index .Results $category) $test).GetEmoji }}</summary>
<ul>
{{ range index (index .Results $category) $test }}<li>{{ . }}</li>{{ end }}
{{ range (index (index .Results $category) $test).Errors }}<li>{{ . }}</li>{{ end }}
</ul>
{{ end }}
</td>{{ end }}
19 changes: 19 additions & 0 deletions test-harness/reports/report-template.junit.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuites time="{{ .TimeTotal | durationToJunit }}">
{{ range $testSuiteName, $tests := .Report.Results }}
<testsuite name="{{ $testSuiteName }}" tests="{{ $tests | len }}" time="{{ index $.TimePerTestSuite $testSuiteName | durationToJunit }}">
{{ range $testCaseName, $result := $tests }}
<testcase name="{{ $testCaseName }}" classname="{{ $testSuiteName }}" time="{{ $result.Time | durationToJunit }}"{{ if $result.Errors }}>
{{ if $result.IsSkipped }}
<skipped message="test is not supported by this SDK" />
{{ else }}
<failure message="test did not pass" type="AssertionError">
{{ range $_, $err := $result.Errors }}{{ $err }}{{ end }}
</failure>
{{ end }}
</testcase>
{{ else }}/>{{ end}}
{{ end }}
</testsuite>
{{ end }}
</testsuites>
2 changes: 1 addition & 1 deletion test-harness/reports/report-template.md
Original file line number Diff line number Diff line change
@@ -8,6 +8,6 @@ SDK: [{{ .TestServerID.Name }}]({{ .TestServerID.Url }}) ({{ .TestServerID.Langu

| Feature | Result |
| ------- | ------ |{{ range $test, $result := $results }}
| {{ $test }} | {{ $result | getEmoji}}{{ if $result }} <ul>{{ range $result }}<li><pre>{{ . | sanatizeHTML }}</pre></li>{{ end }}</ul>{{ end }} |{{ end }}
| {{ $test }} | {{ $result.GetEmoji }}{{ if $result.Errors }} <ul>{{ range $result.Errors }}<li><pre>{{ . | sanatizeHTML }}</pre></li>{{ end }}</ul>{{ end }} |{{ end }}

{{ end }}
2 changes: 1 addition & 1 deletion test-harness/reports/report-template.txt
Original file line number Diff line number Diff line change
@@ -3,5 +3,5 @@ web5 spec conformance report for {{ .TestServerID.Name }} ({{ .TestServerID.Url
{{ range $groupName, $results := .Results }}
{{ $groupName }}
======================{{ range $test, $result := $results }}
{{ $test }}: {{ $result | getEmoji }} {{ if $result }}fail: {{ $result }}{{ else }}pass{{ end }}{{ end }}
{{ $test }}: {{ $result.GetEmoji }} {{ if $result.Errors }}fail: {{ $result.Errors }}{{ else }}pass{{ end }}{{ end }}
{{ end }}
137 changes: 38 additions & 99 deletions test-harness/reports/reports.go
Original file line number Diff line number Diff line change
@@ -1,48 +1,59 @@
package reports

import (
"bytes"
"embed"
"fmt"
"html/template"
"os"
"strings"
"errors"
htmltemplate "html/template"
texttemplate "text/template"
"time"

"golang.org/x/exp/slog"

"github.com/TBD54566975/web5-spec/openapi"
"github.com/TBD54566975/web5-spec/tests"
"github.com/TBD54566975/sdk-development/openapi"
)

//go:embed *
var templatesFS embed.FS
var (
ErrNotSupported = errors.New("test not supported by this SDK")

var templates = template.New("")
//go:embed *
templatesFS embed.FS

htmlTemplates = htmltemplate.New("")
textTemplates = texttemplate.New("")
funcmap = map[string]any{
"sanatizeHTML": sanatizeHTML,
"durationToJunit": durationToJunit,
}
)

func init() {
templates.Funcs(template.FuncMap{
"sanatizeHTML": sanatizeHTML,
"getEmoji": getEmoji,
})
_, err := templates.ParseFS(templatesFS, "report-template.*")
if err != nil {
htmlTemplates.Funcs(funcmap)
if _, err := htmlTemplates.ParseFS(templatesFS, "report-template.html"); err != nil {
panic(err)
}

textTemplates.Funcs(funcmap)
if _, err := textTemplates.ParseFS(templatesFS, "report-template.*"); err != nil {
panic(err)
}
}

type Report struct {
TestServerID openapi.TestServerID
Results map[string]map[string][]error
Results map[string]map[string]Result
}

type Result struct {
Errors []error
Time time.Duration
}

func (r Report) IsPassing() bool {
for _, results := range r.Results {
for _, errs := range results {
if len(errs) == 1 && errs[0] == tests.ErrNotSupported {
for _, result := range results {
if result.IsSkipped() {
continue
}

if len(errs) > 0 {
if len(result.Errors) > 0 {
return false
}
}
@@ -52,90 +63,18 @@ func (r Report) IsPassing() bool {
return true
}

func (r Report) Text() (string, error) {
var buffer bytes.Buffer

if err := templates.ExecuteTemplate(&buffer, "report-template.txt", r); err != nil {
return "", err
}

return buffer.String(), nil
}

func WriteMarkdown(report Report, filename string) error {
f, err := os.Create(filename)
if err != nil {
return err
}
defer f.Close()

err = templates.ExecuteTemplate(f, "report-template.md", report)
if err != nil {
return err
}

return nil
}

func sanatizeHTML(dirty error) string {
clean := strings.ReplaceAll(dirty.Error(), "<", "&lt;")
clean = strings.ReplaceAll(clean, ">", "&gt;")
clean = strings.ReplaceAll(clean, "\n", "\\\\n")

return clean
func (r Result) IsSkipped() bool {
return len(r.Errors) == 1 && r.Errors[0] == ErrNotSupported
}

func getEmoji(errs []error) string {
if len(errs) == 0 {
func (r Result) GetEmoji() string {
if len(r.Errors) == 0 {
return "✅"
}

if len(errs) == 1 && errs[0] == tests.ErrNotSupported {
if len(r.Errors) == 1 && r.Errors[0] == ErrNotSupported {
return "🚧"
}

return "❌"
}

type htmlTemplateInput struct {
Reports []Report
Tests map[string][]string
}

func WriteHTML(reports []Report, filename string) error {
slog.Info("writing html report")

testmap := map[string]map[string]bool{}
for _, report := range reports {
for category, tests := range report.Results {
if _, ok := tests[category]; !ok {
testmap[category] = map[string]bool{}
}

for test := range tests {
testmap[category][test] = true
}
}
}

templateInput := htmlTemplateInput{Reports: reports, Tests: map[string][]string{}}

for category, tests := range testmap {
templateInput.Tests[category] = []string{}
for test := range tests {
templateInput.Tests[category] = append(templateInput.Tests[category], test)
}
}

f, err := os.Create(filename)
if err != nil {
return fmt.Errorf("error opening %s: %v", filename, err)
}
defer f.Close()

if err := templates.ExecuteTemplate(f, "report-template.html", templateInput); err != nil {
return err
}

return nil
}
13 changes: 13 additions & 0 deletions test-harness/reports/text.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package reports

import "bytes"

func (r Report) Text() (string, error) {
var buffer bytes.Buffer

if err := textTemplates.ExecuteTemplate(&buffer, "report-template.txt", r); err != nil {
return "", err
}

return buffer.String(), nil
}
1 change: 0 additions & 1 deletion test-harness/sdks/web5-js/.gitignore

This file was deleted.

53 changes: 0 additions & 53 deletions test-harness/sdks/web5-js/credentials.ts

This file was deleted.

17 changes: 0 additions & 17 deletions test-harness/sdks/web5-js/did-ion.ts

This file was deleted.

64 changes: 0 additions & 64 deletions test-harness/sdks/web5-js/encoders.ts

This file was deleted.

51 changes: 0 additions & 51 deletions test-harness/sdks/web5-js/main.ts

This file was deleted.

498 changes: 0 additions & 498 deletions test-harness/sdks/web5-js/openapi.d.ts

This file was deleted.

2,125 changes: 0 additions & 2,125 deletions test-harness/sdks/web5-js/package-lock.json

This file was deleted.

18 changes: 0 additions & 18 deletions test-harness/sdks/web5-js/package.json

This file was deleted.

27 changes: 0 additions & 27 deletions test-harness/sdks/web5-js/tsconfig.json

This file was deleted.

8 changes: 0 additions & 8 deletions test-harness/sdks/web5-js/web5-spec.Dockerfile

This file was deleted.

5 changes: 4 additions & 1 deletion test-harness/tests/credentials.go
Original file line number Diff line number Diff line change
@@ -5,7 +5,7 @@ import (
"encoding/json"
"log"

"github.com/TBD54566975/web5-spec/openapi"
"github.com/TBD54566975/sdk-development/openapi"
"gopkg.in/square/go-jose.v2"
)

@@ -65,6 +65,9 @@ func vcCreate(ctx context.Context, serverURL string) []error {
vcJwt := response.JSON200.VerifiableCredential.Data

token, err := jose.ParseSigned(vcJwt)
if err != nil {
return []error{err}
}
payloadBytes := token.UnsafePayloadWithoutVerification()

var payload Payload
2 changes: 1 addition & 1 deletion test-harness/tests/did-ion.go
Original file line number Diff line number Diff line change
@@ -5,7 +5,7 @@ import (
"fmt"
"strings"

"github.com/TBD54566975/web5-spec/openapi"
"github.com/TBD54566975/sdk-development/openapi"
)

func init() {
2 changes: 1 addition & 1 deletion test-harness/tests/encoders.go
Original file line number Diff line number Diff line change
@@ -6,7 +6,7 @@ import (
"encoding/base64"
"fmt"

"github.com/TBD54566975/web5-spec/openapi"
"github.com/TBD54566975/sdk-development/openapi"
"github.com/mr-tron/base58"
)

20 changes: 10 additions & 10 deletions test-harness/tests/test-runner.go
Original file line number Diff line number Diff line change
@@ -2,34 +2,34 @@ package tests

import (
"context"
"errors"
"fmt"
"math/rand"
"net/http"
"strings"
"time"

"github.com/TBD54566975/sdk-development/reports"
"golang.org/x/exp/slog"
)

type testfn func(ctx context.Context, serverURL string) []error

var tests = map[string]map[string]testfn{}

var ErrNotSupported = errors.New("test not supported by this SDK")

func RunTests(serverURL string) map[string]map[string][]error {
results := map[string]map[string][]error{}
func RunTests(serverURL string) map[string]map[string]reports.Result {
results := map[string]map[string]reports.Result{}
for group, ts := range tests {
slog.Info("starting test group", "group", group)
results[group] = map[string][]error{}
results[group] = map[string]reports.Result{}
for t, fn := range ts {
slog.Info("running", "test", t)
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
results[group][t] = fn(ctx, serverURL)
if results[t] != nil {
slog.Error("error", "test", t, "error", results[t])
start := time.Now()
errs := fn(ctx, serverURL)
results[group][t] = reports.Result{
Errors: errs,
Time: time.Since(start),
}
}
}
@@ -76,7 +76,7 @@ func compareStringsContains(actual string, expected string, field string) error

func unexpectedResponseCode(r *http.Response, body []byte) []error {
if r.StatusCode == http.StatusNotFound {
return []error{ErrNotSupported}
return []error{reports.ErrNotSupported}
}

return []error{fmt.Errorf("%s: %s", r.Status, string(body))}

0 comments on commit df9d113

Please sign in to comment.