diff --git a/.github/workflows/kind.yaml b/.github/workflows/kind.yaml index e590f934da..01ff1ff034 100644 --- a/.github/workflows/kind.yaml +++ b/.github/workflows/kind.yaml @@ -154,7 +154,15 @@ jobs: # Run the connectivity test in non-default namespace (i.e. not cilium-test) cilium connectivity test --debug --all-flows --test-namespace test-namespace \ --external-from-cidrs="${EXTERNAL_NODE_IPS_PARAM}" \ - --collect-sysdump-on-failure + --collect-sysdump-on-failure --junit-file connectivity-${{ matrix.mode }}.xml + + - name: Upload junit output + if: ${{ always() }} + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce + with: + name: connectivity-${{ matrix.mode }}.xml + path: connectivity-${{ matrix.mode }}.xml + retention-days: 5 - name: Uninstall cilium run: | @@ -197,7 +205,15 @@ jobs: run: | cilium connectivity test --debug --force-deploy --all-flows --test-namespace test-namespace \ --external-from-cidrs="${EXTERNAL_NODE_IPS_PARAM}" \ - --collect-sysdump-on-failure + --collect-sysdump-on-failure --junit-file connectivity-ipsec-${{ matrix.mode }}.xml + + - name: Upload junit output + if: ${{ always() }} + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce + with: + name: connectivity-ipsec-${{ matrix.mode }}.xml + path: connectivity-ipsec-${{ matrix.mode }}.xml + retention-days: 5 - name: Cleanup if: ${{ always() }} @@ -215,7 +231,7 @@ jobs: kubectl label nodes "${node}" cilium.io/no-schedule- done - - name: Upload Artifacts + - name: Upload sysdump if: ${{ !success() }} uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 with: @@ -338,7 +354,16 @@ jobs: - name: Run the multicluster connectivity tests run: | - cilium connectivity test --context $CLUSTER1 --multi-cluster $CLUSTER2 --debug --collect-sysdump-on-failure + cilium connectivity test --context $CLUSTER1 --multi-cluster $CLUSTER2 --debug \ + --collect-sysdump-on-failure --junit-file connectivity-clustermesh.xml + + - name: Upload junit output + if: ${{ always() }} + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce + with: + name: connectivity-clustermesh.xml + path: connectivity-clustermesh.xml + retention-days: 5 - name: Cleanup if: ${{ always() }} diff --git a/connectivity/check/check.go b/connectivity/check/check.go index 78b9e569ad..2adc07c3e6 100644 --- a/connectivity/check/check.go +++ b/connectivity/check/check.go @@ -57,6 +57,7 @@ type Parameters struct { ExternalOtherIP string ExternalFromCIDRs []string ExternalFromCIDRMasks []int // Derived from ExternalFromCIDRs + JunitFile string K8sVersion string HelmChartDirectory string diff --git a/connectivity/check/context.go b/connectivity/check/context.go index fec87f7019..b8700c1e42 100644 --- a/connectivity/check/context.go +++ b/connectivity/check/context.go @@ -6,9 +6,11 @@ package check import ( "bytes" "context" + "errors" "fmt" "net" "net/netip" + "os" "strconv" "strings" "time" @@ -24,6 +26,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/cilium/cilium-cli/defaults" + "github.com/cilium/cilium-cli/internal/junit" "github.com/cilium/cilium-cli/k8s" ) @@ -355,6 +358,10 @@ func (ct *ConnectivityTest) Run(ctx context.Context) error { <-done } + if err := ct.writeJunit(); err != nil { + ct.Failf("writing to junit file %s failed: %s", ct.Params().JunitFile, err) + } + // Report the test results. return ct.report() } @@ -366,6 +373,78 @@ func (ct *ConnectivityTest) skip(t *Test) { t.skipped = true } +func (ct *ConnectivityTest) writeJunit() error { + if ct.Params().JunitFile == "" { + return nil + } + + suite := &junit.TestSuite{ + Name: "connectivity test", + Package: "cilium", + Tests: len(ct.tests), + Properties: &junit.Properties{ + Properties: []junit.Property{ + {Name: "Args", Value: strings.Join(os.Args[3:], "|")}, + }, + }, + } + + for i, t := range ct.tests { + test := &junit.TestCase{ + Name: t.Name(), + Classname: "connectivity test", + Status: "passed", + Time: t.completionTime.Sub(t.startTime).Seconds(), + } + + // Timestamp of the TestSuite is the first test's start time + if i == 0 { + suite.Timestamp = t.startTime.Format("2006-01-02T15:04:05") + } + suite.Time += test.Time + + if t.skipped { + test.Status = "skipped" + test.Skipped = &junit.Skipped{Message: t.Name() + " skipped"} + suite.Skipped++ + test.Time = 0 + } else if t.failed { + test.Status = "failed" + test.Failure = &junit.Failure{Message: t.Name() + " failed", Type: "failure"} + suite.Failures++ + msgs := []string{} + for _, a := range t.failedActions() { + msgs = append(msgs, a.String()) + } + test.Failure.Value = strings.Join(msgs, "\n") + } + + suite.TestCases = append(suite.TestCases, test) + } + + suites := junit.TestSuites{ + Tests: suite.Tests, + Disabled: suite.Skipped, + Failures: suite.Failures, + Time: suite.Time, + TestSuites: []*junit.TestSuite{suite}, + } + + f, err := os.Create(ct.Params().JunitFile) + if err != nil { + return err + } + + if err := suites.WriteReport(f); err != nil { + if e := f.Close(); e != nil { + return errors.Join(err, e) + } + return err + } + + return f.Close() +} + func (ct *ConnectivityTest) report() error { total := ct.tests actions := ct.actions() diff --git a/connectivity/check/test.go b/connectivity/check/test.go index db0b73701d..3fad31c00f 100644 --- a/connectivity/check/test.go +++ b/connectivity/check/test.go @@ -93,6 +93,9 @@ type Test struct { // Start time of the test. startTime time.Time + // Completion time of the test. + completionTime time.Time + // Buffer to store output until it's flushed by a failure. // Unused when run in verbose or debug mode. logMu sync.RWMutex @@ -219,6 +222,10 @@ func (t *Test) Run(ctx context.Context) error { // Store start time of the Test. t.startTime = time.Now() + // Store completion of the Test when function is returned + defer func() { + t.completionTime = time.Now() + }() t.ctx.Logf("[=] Test [%s]", t.Name()) diff --git a/internal/cli/cmd/connectivity.go b/internal/cli/cmd/connectivity.go index dbc3dd8c77..231fb4beae 100644 --- a/internal/cli/cmd/connectivity.go +++ b/internal/cli/cmd/connectivity.go @@ -134,6 +134,7 @@ func newCmdConnectivityTest() *cobra.Command { cmd.Flags().StringVar(¶ms.ExternalIP, "external-ip", "1.1.1.1", "IP to use as external target in connectivity tests") cmd.Flags().StringVar(¶ms.ExternalOtherIP, "external-other-ip", "1.0.0.1", "Other IP to use as external target in connectivity tests") cmd.Flags().StringSliceVar(¶ms.ExternalFromCIDRs, "external-from-cidrs", []string{}, "CIDRs representing nodes without Cilium to be used in connectivity tests") + cmd.Flags().StringVar(¶ms.JunitFile, "junit-file", "", "Generate junit report and write to file") cmd.Flags().BoolVar(¶ms.SkipIPCacheCheck, "skip-ip-cache-check", true, "Skip IPCache check") cmd.Flags().MarkHidden("skip-ip-cache-check") cmd.Flags().BoolVar(¶ms.Datapath, "datapath", false, "Run datapath conformance tests") diff --git a/internal/junit/junit.go b/internal/junit/junit.go new file mode 100644 index 0000000000..fd1a0f6426 --- /dev/null +++ b/internal/junit/junit.go @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Authors of Cilium + +package junit + +import ( + "encoding/xml" + "fmt" + "io" +) + +// TestSuites represents collection of test suites. +type TestSuites struct { + XMLName xml.Name `xml:"testsuites"` + + Tests int `xml:"tests,attr"` + Disabled int `xml:"disabled,attr"` + Errors int `xml:"errors,attr"` + Failures int `xml:"failures,attr"` + Time float64 `xml:"time,attr"` + + TestSuites []*TestSuite `xml:"testsuite"` +} + +// WriteReport writes Junit XML output to a writer interface. +func (ts *TestSuites) WriteReport(w io.Writer) error { + if _, err := fmt.Fprint(w, xml.Header); err != nil { + return err + } + + encoder := xml.NewEncoder(w) + encoder.Indent(" ", " ") + return encoder.Encode(ts) +} + +// TestSuite represents collection of test cases. +type TestSuite struct { + XMLName xml.Name `xml:"testsuite"` + + Name string `xml:"name,attr"` + ID int `xml:"id,attr"` + Package string `xml:"package,attr"` + Tests int `xml:"tests,attr"` + Disabled int `xml:"disabled,attr,omitempty"` + Errors int `xml:"errors,attr"` + Failures int `xml:"failures,attr"` + Skipped int `xml:"skipped,attr,omitempty"` + Time float64 `xml:"time,attr"` + Timestamp string `xml:"timestamp,attr"` + Hostname string `xml:"hostname,attr,omitempty"` + + Properties *Properties `xml:"properties,omitempty"` + + TestCases []*TestCase `xml:"testcase"` + + SystemOut string `xml:"system-out,omitempty"` + SystemErr string `xml:"system-err,omitempty"` +} + +// Properties represents additional information of a test suite. +type Properties struct { + Properties []Property `xml:"property"` +} + +// Property represents a property in Properties. +type Property struct { + XMLName xml.Name `xml:"property"` + + Name string `xml:"name,attr"` + Value string `xml:"value,attr"` +} + +// TestCase represents a single test case within a test suite. +type TestCase struct { + XMLName xml.Name `xml:"testcase"` + + Name string `xml:"name,attr"` + Classname string `xml:"classname,attr"` + Status string `xml:"status,attr,omitempty"` + Time float64 `xml:"time,attr"` + + Skipped *Skipped `xml:"skipped,omitempty"` + Error *Error `xml:"error,omitempty"` + Failure *Failure `xml:"failure,omitempty"` + + SystemOut string `xml:"system-out,omitempty"` + SystemErr string `xml:"system-err,omitempty"` +} + +// Skipped represents a skipped information in a test case. +type Skipped struct { + XMLName xml.Name `xml:"skipped"` + + Message string `xml:"message,attr,omitempty"` +} + +// Error represents an error information in a test case. +type Error struct { + XMLName xml.Name `xml:"error"` + + Message string `xml:"message,attr,omitempty"` + Type string `xml:"type,attr"` + + Value string `xml:",chardata"` +} + +// Failure represents a failure information in a test case. +type Failure struct { + XMLName xml.Name `xml:"failure"` + + Message string `xml:"message,attr,omitempty"` + Type string `xml:"type,attr"` + + Value string `xml:",chardata"` +} diff --git a/internal/junit/junit_test.go b/internal/junit/junit_test.go new file mode 100644 index 0000000000..48e593d079 --- /dev/null +++ b/internal/junit/junit_test.go @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Authors of Cilium + +package junit + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" +) + +var suitesText = ` + + + + + + + + failure > expected + + + ` + +var suites = TestSuites{ + Tests: 3, + Disabled: 1, + Failures: 1, + Time: 14.7, + TestSuites: []*TestSuite{ + { + Name: "testSuite", + Package: "test", + Tests: 3, + Skipped: 1, + Failures: 1, + Time: 14.7, + Timestamp: "2023-05-08T11:24:14", + TestCases: []*TestCase{ + {Name: "test1", Classname: "test", Status: "passed", Time: 7.7}, + {Name: "test2", Classname: "test", Status: "skipped", Time: 0, + Skipped: &Skipped{Message: "test2 skipped"}}, + {Name: "test3", Classname: "test", Status: "failed", Time: 7, + Failure: &Failure{Message: "test3 failed", Type: "failure", Value: "failure > expected"}}, + }, + }, + }, +} + +func TestWriteReport(t *testing.T) { + buf := &bytes.Buffer{} + assert.NoError(t, suites.WriteReport(buf)) + assert.Equal(t, suitesText, buf.String()) +}