Utilize the k8s-reporter in the e2e tests

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/k8snetworkplumbingwg/sriov-network-device-plugin v0.0.0-20221127172732-a5a7395122e3
 	github.com/onsi/ginkgo/v2 v2.9.5
 	github.com/onsi/gomega v1.27.7
+	github.com/openshift-kni/k8sreporter v1.0.4
 	github.com/openshift/api v0.0.0-20221220162201-efeef9d83325
 	github.com/openshift/client-go v0.0.0-20220831193253-4950ae70c8ea
 	github.com/openshift/machine-config-operator v0.0.1-0.20230118083703-fc27a2bdaa85
@@ -35,7 +36,7 @@ require (
 	k8s.io/client-go v0.27.4
 	k8s.io/code-generator v0.27.4
 	k8s.io/kubectl v0.27.4
-	k8s.io/utils v0.0.0-20230209194617-a36077c30491
+	k8s.io/utils v0.0.0-20230313181309-38a27ef9d749
 	sigs.k8s.io/controller-runtime v0.15.2
 )
 

go.sum

@@ -354,6 +354,8 @@ github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
 github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k=
 github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU=
 github.com/onsi/gomega v1.27.7/go.mod h1:1p8OOlwo2iUUDsHnOrjE5UKYJ+e3W8eQ3qSlRahPmr4=
+github.com/openshift-kni/k8sreporter v1.0.4 h1:jEwX6Pqei60kO1U0JLo+ePjQaP7DNn/M6d63KCS2tS0=
+github.com/openshift-kni/k8sreporter v1.0.4/go.mod h1:fg8HI9yxiKAi6UzR6NTtrmQmA2WKzUqmkRUHwQ1+Bj8=
 github.com/openshift/api v0.0.0-20221220162201-efeef9d83325 h1:tUmCk1IW44nT8YjgNCFa6r8lq/jlRrsfb8PLcFEsyb8=
 github.com/openshift/api v0.0.0-20221220162201-efeef9d83325/go.mod h1:OW9hi5XDXOQWm/kRqUww6RVxZSf0nqrS4heerSmHBC4=
 github.com/openshift/client-go v0.0.0-20220831193253-4950ae70c8ea h1:7JbjIzWt3Q75ErY1PAZ+gCA+bErI6HSlpffHFmMMzqM=
@@ -862,8 +864,8 @@ k8s.io/kubectl v0.27.4 h1:RV1TQLIbtL34+vIM+W7HaS3KfAbqvy9lWn6pWB9els4=
 k8s.io/kubectl v0.27.4/go.mod h1:qtc1s3BouB9KixJkriZMQqTsXMc+OAni6FeKAhq7q14=
 k8s.io/kubelet v0.25.1 h1:FBGOmIM4qR4Ov+RU90VXnxO/hvvniUMTGUriVOa9FfY=
 k8s.io/kubelet v0.25.1/go.mod h1:mXo8HjxCrwVduGBk4tzuhegJYPvNwPyycRf39H4KKqE=
-k8s.io/utils v0.0.0-20230209194617-a36077c30491 h1:r0BAOLElQnnFhE/ApUsg3iHdVYYPBjNSSOMowRZxxsY=
-k8s.io/utils v0.0.0-20230209194617-a36077c30491/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+k8s.io/utils v0.0.0-20230313181309-38a27ef9d749 h1:xMMXJlJbsU8w3V5N2FLDQ8YgU8s1EoULdbQBcAeNJkY=
+k8s.io/utils v0.0.0-20230313181309-38a27ef9d749/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

hack/run-e2e-conformance.sh

@@ -18,4 +18,4 @@ GOPATH="${GOPATH:-~/go}"
 JUNIT_OUTPUT="${JUNIT_OUTPUT:-/tmp/artifacts}"
 export PATH=$PATH:$GOPATH/bin
 
-GOFLAGS=-mod=vendor ginkgo -output-dir=$JUNIT_OUTPUT --junit-report "unit_report.xml" "$SUITE"
+GOFLAGS=-mod=vendor ginkgo -output-dir=$JUNIT_OUTPUT --junit-report "unit_report.xml" "$SUITE" -- -report=$JUNIT_OUTPUT

test/conformance/test_suite_test.go

@@ -2,81 +2,60 @@ package conformance
 
 import (
 	"flag"
-	"fmt"
-	"os"
+	"log"
 	"path"
 	"testing"
-
-	"github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/clean"
+	"time"
 
 	. "github.com/onsi/ginkgo/v2"
-	"github.com/onsi/ginkgo/v2/reporters"
-	"github.com/onsi/ginkgo/v2/types"
 	. "github.com/onsi/gomega"
 
-	testclient "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/client"
+	kniK8sReporter "github.com/openshift-kni/k8sreporter"
 
 	// Test files in this package must not end with `_test.go` suffix, as they are imported as go package
 	_ "github.com/k8snetworkplumbingwg/sriov-network-operator/test/conformance/tests"
 
+	"github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/clean"
 	"github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/k8sreporter"
 )
 
 var (
-	junitPath      *string
-	dumpOutput     *bool
-	reporterFile   string
-	customReporter *k8sreporter.KubernetesReporter
+	customReporter *kniK8sReporter.KubernetesReporter
+	err            error
+	reportPath     *string
 )
 
 func init() {
-	dumpOutput = flag.Bool("dump", false, "dump informations for failed tests")
-	junitPath = flag.String("junit", "", "the path for the junit format report")
+	reportPath = flag.String("report", "", "the path of the report directory containing details for failed tests")
 }
 
 func TestTest(t *testing.T) {
-	RegisterFailHandler(Fail)
-
-	reporterFile = os.Getenv("REPORTER_OUTPUT")
-
-	clients := testclient.New("")
+	// We want to collect logs before any resource is deleted in AfterEach, so we register the global fail handler
+	// in a way such that the reporter's Dump is always called before the default Fail.
+	RegisterFailHandler(
+		func(message string, callerSkip ...int) {
+			if customReporter != nil {
+				customReporter.Dump(10*time.Minute, CurrentSpecReport().FullText())
+			}
+
+			// Ensure failing line location is not affected by this wrapper
+			for i := range callerSkip {
+				callerSkip[i]++
+			}
+			Fail(message, callerSkip...)
+		})
 
-	if reporterFile != "" {
-		f, err := os.OpenFile(reporterFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if *reportPath != "" {
+		reportFile := path.Join(*reportPath, "sriov_failure_report.log")
+		customReporter, err = k8sreporter.New(reportFile)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "failed to open the file: %v\n", err)
-			return
+			log.Fatalf("Failed to create the k8s reporter %s", err)
 		}
-		defer f.Close()
-		customReporter = k8sreporter.New(clients, f)
-	} else if *dumpOutput {
-		customReporter = k8sreporter.New(clients, os.Stdout)
 	}
 
 	RunSpecs(t, "SRIOV Operator conformance tests")
 }
 
-var _ = ReportAfterSuite("conformance", func(report types.Report) {
-	if *junitPath != "" {
-		junitFile := path.Join(*junitPath, "junit_sriov_conformance.xml")
-		reporters.GenerateJUnitReportWithConfig(report, junitFile, reporters.JunitReportConfig{
-			OmitTimelinesForSpecState: types.SpecStatePassed | types.SpecStateSkipped,
-			OmitLeafNodeType:          true,
-			OmitSuiteSetupNodes:       true,
-		})
-	}
-})
-
-var _ = ReportAfterEach(func(sr types.SpecReport) {
-	if sr.Failed() == false {
-		return
-	}
-
-	if reporterFile != "" || *dumpOutput {
-		customReporter.Report(sr)
-	}
-})
-
 var _ = BeforeSuite(func() {
 	err := clean.All()
 	Expect(err).NotTo(HaveOccurred())

test/util/k8sreporter/reporter.go

@@ -1,144 +1,53 @@
 package k8sreporter
 
 import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
+	"errors"
 	"os"
 	"strings"
-	"sync"
 
-	"github.com/onsi/ginkgo/v2/types"
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
+	kniK8sReporter "github.com/openshift-kni/k8sreporter"
+	"k8s.io/apimachinery/pkg/runtime"
 
 	sriovv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
-	testclient "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/client"
 	"github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/namespaces"
 )
 
-type KubernetesReporter struct {
-	sync.Mutex
-	clients    *testclient.ClientSet
-	dumpOutput io.Writer
-}
-
-func New(clients *testclient.ClientSet, dumpDestination io.Writer) *KubernetesReporter {
-	return &KubernetesReporter{clients: clients, dumpOutput: dumpDestination}
-}
-
-func (r *KubernetesReporter) Report(sr types.SpecReport) {
-	r.Lock()
-	defer r.Unlock()
-	fmt.Fprintln(r.dumpOutput, "Starting dump for failed spec", sr.ContainerHierarchyTexts)
-	r.dump()
-	fmt.Fprintln(r.dumpOutput, "Finished dump for failed spec")
-}
-
-func (r *KubernetesReporter) dump() {
-	r.logNodes()
-	r.logPods("openshift-sriov-network-operator")
-	r.logPods(namespaces.Test)
-	r.logLogs(func(p *corev1.Pod) bool {
-		return !strings.HasPrefix(p.Name, "sriov-")
-	})
-	r.logSriovNodeState()
-	r.logNetworkPolicies()
-}
-
-func (r *KubernetesReporter) logPods(namespace string) {
-	fmt.Fprintf(r.dumpOutput, "Logging pods for %s", namespace)
-
-	pods, err := r.clients.Pods(namespace).List(context.Background(), metav1.ListOptions{})
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "failed to fetch pods: %v\n", err)
-		return
-	}
-
-	j, err := json.MarshalIndent(pods, "", "    ")
-	if err != nil {
-		fmt.Println("Failed to marshal pods", err)
-		return
-	}
-	fmt.Fprintln(r.dumpOutput, string(j))
-}
-
-func (r *KubernetesReporter) logNodes() {
-	fmt.Fprintf(r.dumpOutput, "Logging nodes")
-
-	nodes, err := r.clients.CoreV1Interface.Nodes().List(context.Background(), metav1.ListOptions{})
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "failed to fetch nodes: %v\n", err)
-		return
-	}
-
-	j, err := json.MarshalIndent(nodes, "", "    ")
-	if err != nil {
-		fmt.Println("Failed to marshal nodes")
-		return
-	}
-	fmt.Fprintln(r.dumpOutput, string(j))
-}
-
-func (r *KubernetesReporter) logLogs(filterPods func(*corev1.Pod) bool) {
-	fmt.Fprintf(r.dumpOutput, "Logging pods logs")
-
-	pods, err := r.clients.Pods(corev1.NamespaceAll).List(context.Background(), metav1.ListOptions{})
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "failed to fetch pods: %v\n", err)
-		return
-	}
-
-	for _, pod := range pods.Items {
-		if filterPods(&pod) {
-			continue
-		}
-		for _, container := range pod.Spec.Containers {
-			logs, err := r.clients.Pods(pod.Namespace).GetLogs(pod.Name, &corev1.PodLogOptions{Container: container.Name}).DoRaw(context.Background())
-			if err == nil {
-				fmt.Fprintf(r.dumpOutput, "Dumping logs for pod %s-%s-%s", pod.Namespace, pod.Name, container.Name)
-				fmt.Fprintln(r.dumpOutput, string(logs))
-			}
+func New(reportPath string) (*kniK8sReporter.KubernetesReporter, error) {
+	addToScheme := func(s *runtime.Scheme) error {
+		err := sriovv1.AddToScheme(s)
+		if err != nil {
+			return err
 		}
+		return nil
 	}
-}
-
-func (r *KubernetesReporter) logNetworkPolicies() {
-	fmt.Fprintf(r.dumpOutput, "Logging network policies")
-
-	policies := sriovv1.SriovNetworkNodePolicyList{}
-	err := r.clients.List(context.Background(),
-		&policies,
-		runtimeclient.InNamespace("openshift-sriov-network-operator"))
 
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "failed to fetch network policies: %v\n", err)
-		return
+	dumpNamespace := func(ns string) bool {
+		switch {
+		case ns == namespaces.Test:
+			return true
+		case ns == "openshift-sriov-network-operator":
+			return true
+		case strings.HasPrefix(ns, "sriov-"):
+			return true
+		}
+		return false
 	}
 
-	j, err := json.MarshalIndent(policies, "", "    ")
-	if err != nil {
-		fmt.Println("Failed to marshal policies")
-		return
+	crds := []kniK8sReporter.CRData{
+		{Cr: &sriovv1.SriovNetworkNodeStateList{}},
+		{Cr: &sriovv1.SriovNetworkNodePolicyList{}},
+		{Cr: &sriovv1.SriovNetworkList{}},
+		{Cr: &sriovv1.SriovOperatorConfigList{}},
 	}
-	fmt.Fprintln(r.dumpOutput, string(j))
-}
 
-func (r *KubernetesReporter) logSriovNodeState() {
-	fmt.Fprintf(r.dumpOutput, "Logging node states")
-
-	nodeStates, err := r.clients.SriovNetworkNodeStates("openshift-sriov-network-operator").List(context.Background(), metav1.ListOptions{})
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "failed to fetch node states: %v\n", err)
-		return
+	err := os.Mkdir(reportPath, 0755)
+	if err != nil && !errors.Is(err, os.ErrExist) {
+		return nil, err
 	}
 
-	j, err := json.MarshalIndent(nodeStates, "", "    ")
+	reporter, err := kniK8sReporter.New("", addToScheme, dumpNamespace, reportPath, crds...)
 	if err != nil {
-		fmt.Println("Failed to marshal node states")
-		return
+		return nil, err
 	}
-	fmt.Fprintln(r.dumpOutput, string(j))
+	return reporter, nil
 }