Skip to content

Commit

Permalink
connectivity test: add junit output
Browse files Browse the repository at this point in the history
This commit adds '--junit-file' option to connectivity test
and when option is provided connectivity test output
is written to the destination file as junit xml format.

Signed-off-by: Birol Bilgin <[email protected]>
  • Loading branch information
brlbil committed May 9, 2023
1 parent 75bcca2 commit a235918
Show file tree
Hide file tree
Showing 7 changed files with 284 additions and 4 deletions.
33 changes: 29 additions & 4 deletions .github/workflows/kind.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,15 @@ jobs:
# Run the connectivity test in non-default namespace (i.e. not cilium-test)
cilium connectivity test --debug --all-flows --test-namespace test-namespace \
--external-from-cidrs="${EXTERNAL_NODE_IPS_PARAM}" \
--collect-sysdump-on-failure
--collect-sysdump-on-failure --junit-file connectivity-${{ matrix.mode }}.xml
- name: Upload junit output
if: ${{ always() }}
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
with:
name: connectivity-${{ matrix.mode }}.xml
path: connectivity-${{ matrix.mode }}.xml
retention-days: 5

- name: Uninstall cilium
run: |
Expand Down Expand Up @@ -197,7 +205,15 @@ jobs:
run: |
cilium connectivity test --debug --force-deploy --all-flows --test-namespace test-namespace \
--external-from-cidrs="${EXTERNAL_NODE_IPS_PARAM}" \
--collect-sysdump-on-failure
--collect-sysdump-on-failure --junit-file connectivity-ipsec-${{ matrix.mode }}.xml
- name: Upload junit output
if: ${{ always() }}
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
with:
name: connectivity-ipsec-${{ matrix.mode }}.xml
path: connectivity-ipsec-${{ matrix.mode }}.xml
retention-days: 5

- name: Cleanup
if: ${{ always() }}
Expand All @@ -215,7 +231,7 @@ jobs:
kubectl label nodes "${node}" cilium.io/no-schedule-
done
- name: Upload Artifacts
- name: Upload sysdump
if: ${{ !success() }}
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
with:
Expand Down Expand Up @@ -338,7 +354,16 @@ jobs:
- name: Run the multicluster connectivity tests
run: |
cilium connectivity test --context $CLUSTER1 --multi-cluster $CLUSTER2 --debug --collect-sysdump-on-failure
cilium connectivity test --context $CLUSTER1 --multi-cluster $CLUSTER2 --debug \
--collect-sysdump-on-failure --junit-file connectivity-clustermesh.xml
- name: Upload junit output
if: ${{ always() }}
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
with:
name: connectivity-clustermesh.xml
path: connectivity-clustermesh.xml
retention-days: 5

- name: Cleanup
if: ${{ always() }}
Expand Down
1 change: 1 addition & 0 deletions connectivity/check/check.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ type Parameters struct {
ExternalOtherIP string
ExternalFromCIDRs []string
ExternalFromCIDRMasks []int // Derived from ExternalFromCIDRs
JunitFile string

K8sVersion string
HelmChartDirectory string
Expand Down
75 changes: 75 additions & 0 deletions connectivity/check/context.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"fmt"
"net"
"net/netip"
"os"
"strconv"
"strings"
"time"
Expand All @@ -24,6 +25,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"github.com/cilium/cilium-cli/defaults"
"github.com/cilium/cilium-cli/internal/junit"
"github.com/cilium/cilium-cli/k8s"
)

Expand Down Expand Up @@ -355,6 +357,10 @@ func (ct *ConnectivityTest) Run(ctx context.Context) error {
<-done
}

if err := ct.writeJunit(); err != nil {
ct.Fail(err)
}

// Report the test results.
return ct.report()
}
Expand All @@ -366,6 +372,75 @@ func (ct *ConnectivityTest) skip(t *Test) {
t.skipped = true
}

func (ct *ConnectivityTest) writeJunit() error {
if ct.Params().JunitFile == "" {
return nil
}

suite := &junit.TestSuite{
Name: "connectivity test",
Package: "cilium",
Tests: len(ct.tests),
Properties: &junit.Properties{
Properties: []junit.Property{
{Name: "Args", Value: strings.Join(os.Args[3:], "|")},
},
},
}

for i, t := range ct.tests {
test := &junit.TestCase{
Name: t.Name(),
Classname: "connectivity test",
Status: "passed",
Time: t.completionTime.Sub(t.startTime).Seconds(),
}

// Timestamp of the TestSuite is the first test's start time
if i == 0 {
suite.Timestamp = t.startTime.Format("2006-01-02T15:04:05")
}
suite.Time += test.Time

if t.skipped {
test.Status = "skipped"
test.Skipped = &junit.Skipped{Message: t.Name() + " skipped"}
suite.Skipped++
test.Time = 0
} else if t.failed {
test.Status = "failed"
test.Failure = &junit.Failure{Message: t.Name() + " failed", Type: "failure"}
suite.Failures++
msgs := []string{}
for _, a := range t.failedActions() {
msgs = append(msgs, a.String())
}
test.Failure.Value = strings.Join(msgs, "\n")
}

suite.TestCases = append(suite.TestCases, test)
}

suites := junit.TestSuites{
Tests: suite.Tests,
Disabled: suite.Skipped,
Failures: suite.Failures,
Time: suite.Time,
TestSuites: []*junit.TestSuite{suite},
}

f, err := os.Create(ct.Params().JunitFile)
if err != nil {
return err
}

if err := suites.WriteReport(f); err != nil {
return err
}

return f.Close()
}

func (ct *ConnectivityTest) report() error {
total := ct.tests
actions := ct.actions()
Expand Down
5 changes: 5 additions & 0 deletions connectivity/check/test.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,9 @@ type Test struct {
// Start time of the test.
startTime time.Time

// Completion time of the test.
completionTime time.Time

// Buffer to store output until it's flushed by a failure.
// Unused when run in verbose or debug mode.
logMu sync.RWMutex
Expand Down Expand Up @@ -245,6 +248,8 @@ func (t *Test) Run(ctx context.Context) error {
s.Run(ctx, t)
}

t.completionTime = time.Now()

if t.logBuf != nil {
fmt.Fprintln(t.ctx.params.Writer)
}
Expand Down
1 change: 1 addition & 0 deletions internal/cli/cmd/connectivity.go
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@ func newCmdConnectivityTest() *cobra.Command {
cmd.Flags().StringVar(&params.ExternalIP, "external-ip", "1.1.1.1", "IP to use as external target in connectivity tests")
cmd.Flags().StringVar(&params.ExternalOtherIP, "external-other-ip", "1.0.0.1", "Other IP to use as external target in connectivity tests")
cmd.Flags().StringSliceVar(&params.ExternalFromCIDRs, "external-from-cidrs", []string{}, "CIDRs representing nodes without Cilium to be used in connectivity tests")
cmd.Flags().StringVar(&params.JunitFile, "junit-file", "", "Generate junit report and write to file")
cmd.Flags().BoolVar(&params.SkipIPCacheCheck, "skip-ip-cache-check", true, "Skip IPCache check")
cmd.Flags().MarkHidden("skip-ip-cache-check")
cmd.Flags().BoolVar(&params.Datapath, "datapath", false, "Run datapath conformance tests")
Expand Down
115 changes: 115 additions & 0 deletions internal/junit/junit.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
// SPDX-License-Identifier: Apache-2.0
// Copyright Authors of Cilium

package junit

import (
"encoding/xml"
"fmt"
"io"
)

// TestSuites represents collection of test suites.
type TestSuites struct {
XMLName xml.Name `xml:"testsuites"`

Tests int `xml:"tests,attr"`
Disabled int `xml:"disabled,attr"`
Errors int `xml:"errors,attr"`
Failures int `xml:"failures,attr"`
Time float64 `xml:"time,attr"`

TestSuites []*TestSuite `xml:"testsuite"`
}

// WriteReport writes Junit XML output to a writer interface.
func (ts *TestSuites) WriteReport(w io.Writer) error {
if _, err := fmt.Fprint(w, xml.Header); err != nil {
return err
}

encoder := xml.NewEncoder(w)
encoder.Indent(" ", " ")
return encoder.Encode(ts)
}

// TestSuite represents collection of test cases.
type TestSuite struct {
XMLName xml.Name `xml:"testsuite"`

Name string `xml:"name,attr"`
ID int `xml:"id,attr"`
Package string `xml:"package,attr"`
Tests int `xml:"tests,attr"`
Disabled int `xml:"disabled,attr,omitempty"`
Errors int `xml:"errors,attr"`
Failures int `xml:"failures,attr"`
Skipped int `xml:"skipped,attr,omitempty"`
Time float64 `xml:"time,attr"`
Timestamp string `xml:"timestamp,attr"`
Hostname string `xml:"hostname,attr,omitempty"`

Properties *Properties `xml:"properties,omitempty"`

TestCases []*TestCase `xml:"testcase"`

SystemOut string `xml:"system-out,omitempty"`
SystemErr string `xml:"system-err,omitempty"`
}

// Properties represents additional information of a test suite.
type Properties struct {
Properties []Property `xml:"property"`
}

// Property represents a property in Properties.
type Property struct {
XMLName xml.Name `xml:"property"`

Name string `xml:"name,attr"`
Value string `xml:"value,attr"`
}

// TestCase represents a single test case within a test suite.
type TestCase struct {
XMLName xml.Name `xml:"testcase"`

Name string `xml:"name,attr"`
Classname string `xml:"classname,attr"`
Status string `xml:"status,attr,omitempty"`
Time float64 `xml:"time,attr"`

Skipped *Skipped `xml:"skipped,omitempty"`
Error *Error `xml:"error,omitempty"`
Failure *Failure `xml:"failure,omitempty"`

SystemOut string `xml:"system-out,omitempty"`
SystemErr string `xml:"system-err,omitempty"`
}

// Skipped represents a skipped information in a test case.
type Skipped struct {
XMLName xml.Name `xml:"skipped"`

Message string `xml:"message,attr,omitempty"`
}

// Error represents an error information in a test case.
type Error struct {
XMLName xml.Name `xml:"error"`

Message string `xml:"message,attr,omitempty"`
Type string `xml:"type,attr"`

Value string `xml:",chardata"`
}

// Failure represents a failure information in a test case.
type Failure struct {
XMLName xml.Name `xml:"failure"`

Message string `xml:"message,attr,omitempty"`
Type string `xml:"type,attr"`

Value string `xml:",chardata"`
}
58 changes: 58 additions & 0 deletions internal/junit/junit_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
// SPDX-License-Identifier: Apache-2.0
// Copyright Authors of Cilium

package junit

import (
"bytes"
"testing"
)

var suitesText = `<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="3" disabled="1" errors="0" failures="1" time="14.7">
<testsuite name="testSuite" id="0" package="test" tests="3" errors="0" failures="1" skipped="1" time="14.7" timestamp="2023-05-08T11:24:14">
<testcase name="test1" classname="test" status="passed" time="7.7"></testcase>
<testcase name="test2" classname="test" status="skipped" time="0">
<skipped message="test2 skipped"></skipped>
</testcase>
<testcase name="test3" classname="test" status="failed" time="7">
<failure message="test3 failed" type="failure">failure &gt; expected</failure>
</testcase>
</testsuite>
</testsuites>`

var suites = TestSuites{
Tests: 3,
Disabled: 1,
Failures: 1,
Time: 14.7,
TestSuites: []*TestSuite{
{
Name: "testSuite",
Package: "test",
Tests: 3,
Skipped: 1,
Failures: 1,
Time: 14.7,
Timestamp: "2023-05-08T11:24:14",
TestCases: []*TestCase{
{Name: "test1", Classname: "test", Status: "passed", Time: 7.7},
{Name: "test2", Classname: "test", Status: "skipped", Time: 0,
Skipped: &Skipped{Message: "test2 skipped"}},
{Name: "test3", Classname: "test", Status: "failed", Time: 7,
Failure: &Failure{Message: "test3 failed", Type: "failure", Value: "failure > expected"}},
},
},
},
}

func TestWriteReport(t *testing.T) {
buf := &bytes.Buffer{}
if err := suites.WriteReport(buf); err != nil {
t.Fatalf("Expected error <nil>, got %s", err)
}

if out := buf.String(); out != suitesText {
t.Errorf("Expected output '%s', is not equal to '%s'", suitesText, out)
}
}

0 comments on commit a235918

Please sign in to comment.