Skip to content

Commit

Permalink
Utilize the k8s-reporter in the e2e tests
Browse files Browse the repository at this point in the history
Modify the k8sreporter/reporter.go to utilize
the latest external k8s-reporter package.

The new reporter still outputs the same output as before
plus the CRDs: SriovOperatorConfig and SriovNetwork.

It is used by the conformance and validation suites,
consumes the flag "-report" as the path to where the dir
"sriov_failure_report.log" will be created and hold all of
the failed tests logs.
Also removed the unused `-junit` flag, as we're using Ginkgo's
built it --junit-report tool.

Added the `-report` flag to the run-e2e-conformance.sh script.

Signed-off-by: Lior Noy <[email protected]>
  • Loading branch information
liornoy committed Sep 12, 2023
1 parent 73e7564 commit 992723c
Show file tree
Hide file tree
Showing 15 changed files with 656 additions and 207 deletions.
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ require (
github.com/k8snetworkplumbingwg/sriov-network-device-plugin v0.0.0-20221127172732-a5a7395122e3
github.com/onsi/ginkgo/v2 v2.9.5
github.com/onsi/gomega v1.27.7
github.com/openshift-kni/k8sreporter v1.0.4
github.com/openshift/api v0.0.0-20221220162201-efeef9d83325
github.com/openshift/client-go v0.0.0-20220831193253-4950ae70c8ea
github.com/openshift/machine-config-operator v0.0.1-0.20230118083703-fc27a2bdaa85
Expand All @@ -35,7 +36,7 @@ require (
k8s.io/client-go v0.27.4
k8s.io/code-generator v0.27.4
k8s.io/kubectl v0.27.4
k8s.io/utils v0.0.0-20230209194617-a36077c30491
k8s.io/utils v0.0.0-20230313181309-38a27ef9d749
sigs.k8s.io/controller-runtime v0.15.2
)

Expand Down
6 changes: 4 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -354,6 +354,8 @@ github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k=
github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU=
github.com/onsi/gomega v1.27.7/go.mod h1:1p8OOlwo2iUUDsHnOrjE5UKYJ+e3W8eQ3qSlRahPmr4=
github.com/openshift-kni/k8sreporter v1.0.4 h1:jEwX6Pqei60kO1U0JLo+ePjQaP7DNn/M6d63KCS2tS0=
github.com/openshift-kni/k8sreporter v1.0.4/go.mod h1:fg8HI9yxiKAi6UzR6NTtrmQmA2WKzUqmkRUHwQ1+Bj8=
github.com/openshift/api v0.0.0-20221220162201-efeef9d83325 h1:tUmCk1IW44nT8YjgNCFa6r8lq/jlRrsfb8PLcFEsyb8=
github.com/openshift/api v0.0.0-20221220162201-efeef9d83325/go.mod h1:OW9hi5XDXOQWm/kRqUww6RVxZSf0nqrS4heerSmHBC4=
github.com/openshift/client-go v0.0.0-20220831193253-4950ae70c8ea h1:7JbjIzWt3Q75ErY1PAZ+gCA+bErI6HSlpffHFmMMzqM=
Expand Down Expand Up @@ -862,8 +864,8 @@ k8s.io/kubectl v0.27.4 h1:RV1TQLIbtL34+vIM+W7HaS3KfAbqvy9lWn6pWB9els4=
k8s.io/kubectl v0.27.4/go.mod h1:qtc1s3BouB9KixJkriZMQqTsXMc+OAni6FeKAhq7q14=
k8s.io/kubelet v0.25.1 h1:FBGOmIM4qR4Ov+RU90VXnxO/hvvniUMTGUriVOa9FfY=
k8s.io/kubelet v0.25.1/go.mod h1:mXo8HjxCrwVduGBk4tzuhegJYPvNwPyycRf39H4KKqE=
k8s.io/utils v0.0.0-20230209194617-a36077c30491 h1:r0BAOLElQnnFhE/ApUsg3iHdVYYPBjNSSOMowRZxxsY=
k8s.io/utils v0.0.0-20230209194617-a36077c30491/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
k8s.io/utils v0.0.0-20230313181309-38a27ef9d749 h1:xMMXJlJbsU8w3V5N2FLDQ8YgU8s1EoULdbQBcAeNJkY=
k8s.io/utils v0.0.0-20230313181309-38a27ef9d749/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
Expand Down
2 changes: 1 addition & 1 deletion hack/run-e2e-conformance.sh
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,4 @@ GOPATH="${GOPATH:-~/go}"
JUNIT_OUTPUT="${JUNIT_OUTPUT:-/tmp/artifacts}"
export PATH=$PATH:$GOPATH/bin

GOFLAGS=-mod=vendor ginkgo -output-dir=$JUNIT_OUTPUT --junit-report "unit_report.xml" "$SUITE"
GOFLAGS=-mod=vendor ginkgo -output-dir=$JUNIT_OUTPUT --junit-report "unit_report.xml" "$SUITE" -- -report=$JUNIT_OUTPUT
73 changes: 26 additions & 47 deletions test/conformance/test_suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,81 +2,60 @@ package conformance

import (
"flag"
"fmt"
"os"
"log"
"path"
"testing"

"github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/clean"
"time"

. "github.com/onsi/ginkgo/v2"
"github.com/onsi/ginkgo/v2/reporters"
"github.com/onsi/ginkgo/v2/types"
. "github.com/onsi/gomega"

testclient "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/client"
kniK8sReporter "github.com/openshift-kni/k8sreporter"

// Test files in this package must not end with `_test.go` suffix, as they are imported as go package
_ "github.com/k8snetworkplumbingwg/sriov-network-operator/test/conformance/tests"

"github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/clean"
"github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/k8sreporter"
)

var (
junitPath *string
dumpOutput *bool
reporterFile string
customReporter *k8sreporter.KubernetesReporter
customReporter *kniK8sReporter.KubernetesReporter
err error
reportPath *string
)

func init() {
dumpOutput = flag.Bool("dump", false, "dump informations for failed tests")
junitPath = flag.String("junit", "", "the path for the junit format report")
reportPath = flag.String("report", "", "the path of the report directory containing details for failed tests")
}

func TestTest(t *testing.T) {
RegisterFailHandler(Fail)

reporterFile = os.Getenv("REPORTER_OUTPUT")

clients := testclient.New("")
// We want to collect logs before any resource is deleted in AfterEach, so we register the global fail handler
// in a way such that the reporter's Dump is always called before the default Fail.
RegisterFailHandler(
func(message string, callerSkip ...int) {
if customReporter != nil {
customReporter.Dump(10*time.Minute, CurrentSpecReport().FullText())
}

// Ensure failing line location is not affected by this wrapper
for i := range callerSkip {
callerSkip[i]++
}
Fail(message, callerSkip...)
})

if reporterFile != "" {
f, err := os.OpenFile(reporterFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if *reportPath != "" {
reportFile := path.Join(*reportPath, "sriov_failure_report.log")
customReporter, err = k8sreporter.New(reportFile)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to open the file: %v\n", err)
return
log.Fatalf("Failed to create the k8s reporter %s", err)
}
defer f.Close()
customReporter = k8sreporter.New(clients, f)
} else if *dumpOutput {
customReporter = k8sreporter.New(clients, os.Stdout)
}

RunSpecs(t, "SRIOV Operator conformance tests")
}

var _ = ReportAfterSuite("conformance", func(report types.Report) {
if *junitPath != "" {
junitFile := path.Join(*junitPath, "junit_sriov_conformance.xml")
reporters.GenerateJUnitReportWithConfig(report, junitFile, reporters.JunitReportConfig{
OmitTimelinesForSpecState: types.SpecStatePassed | types.SpecStateSkipped,
OmitLeafNodeType: true,
OmitSuiteSetupNodes: true,
})
}
})

var _ = ReportAfterEach(func(sr types.SpecReport) {
if sr.Failed() == false {
return
}

if reporterFile != "" || *dumpOutput {
customReporter.Report(sr)
}
})

var _ = BeforeSuite(func() {
err := clean.All()
Expect(err).NotTo(HaveOccurred())
Expand Down
151 changes: 30 additions & 121 deletions test/util/k8sreporter/reporter.go
Original file line number Diff line number Diff line change
@@ -1,144 +1,53 @@
package k8sreporter

import (
"context"
"encoding/json"
"fmt"
"io"
"errors"
"os"
"strings"
"sync"

"github.com/onsi/ginkgo/v2/types"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
kniK8sReporter "github.com/openshift-kni/k8sreporter"
"k8s.io/apimachinery/pkg/runtime"

sriovv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
testclient "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/client"
"github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/namespaces"
)

type KubernetesReporter struct {
sync.Mutex
clients *testclient.ClientSet
dumpOutput io.Writer
}

func New(clients *testclient.ClientSet, dumpDestination io.Writer) *KubernetesReporter {
return &KubernetesReporter{clients: clients, dumpOutput: dumpDestination}
}

func (r *KubernetesReporter) Report(sr types.SpecReport) {
r.Lock()
defer r.Unlock()
fmt.Fprintln(r.dumpOutput, "Starting dump for failed spec", sr.ContainerHierarchyTexts)
r.dump()
fmt.Fprintln(r.dumpOutput, "Finished dump for failed spec")
}

func (r *KubernetesReporter) dump() {
r.logNodes()
r.logPods("openshift-sriov-network-operator")
r.logPods(namespaces.Test)
r.logLogs(func(p *corev1.Pod) bool {
return !strings.HasPrefix(p.Name, "sriov-")
})
r.logSriovNodeState()
r.logNetworkPolicies()
}

func (r *KubernetesReporter) logPods(namespace string) {
fmt.Fprintf(r.dumpOutput, "Logging pods for %s", namespace)

pods, err := r.clients.Pods(namespace).List(context.Background(), metav1.ListOptions{})
if err != nil {
fmt.Fprintf(os.Stderr, "failed to fetch pods: %v\n", err)
return
}

j, err := json.MarshalIndent(pods, "", " ")
if err != nil {
fmt.Println("Failed to marshal pods", err)
return
}
fmt.Fprintln(r.dumpOutput, string(j))
}

func (r *KubernetesReporter) logNodes() {
fmt.Fprintf(r.dumpOutput, "Logging nodes")

nodes, err := r.clients.CoreV1Interface.Nodes().List(context.Background(), metav1.ListOptions{})
if err != nil {
fmt.Fprintf(os.Stderr, "failed to fetch nodes: %v\n", err)
return
}

j, err := json.MarshalIndent(nodes, "", " ")
if err != nil {
fmt.Println("Failed to marshal nodes")
return
}
fmt.Fprintln(r.dumpOutput, string(j))
}

func (r *KubernetesReporter) logLogs(filterPods func(*corev1.Pod) bool) {
fmt.Fprintf(r.dumpOutput, "Logging pods logs")

pods, err := r.clients.Pods(corev1.NamespaceAll).List(context.Background(), metav1.ListOptions{})
if err != nil {
fmt.Fprintf(os.Stderr, "failed to fetch pods: %v\n", err)
return
}

for _, pod := range pods.Items {
if filterPods(&pod) {
continue
}
for _, container := range pod.Spec.Containers {
logs, err := r.clients.Pods(pod.Namespace).GetLogs(pod.Name, &corev1.PodLogOptions{Container: container.Name}).DoRaw(context.Background())
if err == nil {
fmt.Fprintf(r.dumpOutput, "Dumping logs for pod %s-%s-%s", pod.Namespace, pod.Name, container.Name)
fmt.Fprintln(r.dumpOutput, string(logs))
}
func New(reportPath string) (*kniK8sReporter.KubernetesReporter, error) {
addToScheme := func(s *runtime.Scheme) error {
err := sriovv1.AddToScheme(s)
if err != nil {
return err
}
return nil
}
}

func (r *KubernetesReporter) logNetworkPolicies() {
fmt.Fprintf(r.dumpOutput, "Logging network policies")

policies := sriovv1.SriovNetworkNodePolicyList{}
err := r.clients.List(context.Background(),
&policies,
runtimeclient.InNamespace("openshift-sriov-network-operator"))

if err != nil {
fmt.Fprintf(os.Stderr, "failed to fetch network policies: %v\n", err)
return
dumpNamespace := func(ns string) bool {
switch {
case ns == namespaces.Test:
return true
case ns == "openshift-sriov-network-operator":
return true
case strings.HasPrefix(ns, "sriov-"):
return true
}
return false
}

j, err := json.MarshalIndent(policies, "", " ")
if err != nil {
fmt.Println("Failed to marshal policies")
return
crds := []kniK8sReporter.CRData{
{Cr: &sriovv1.SriovNetworkNodeStateList{}},
{Cr: &sriovv1.SriovNetworkNodePolicyList{}},
{Cr: &sriovv1.SriovNetworkList{}},
{Cr: &sriovv1.SriovOperatorConfigList{}},
}
fmt.Fprintln(r.dumpOutput, string(j))
}

func (r *KubernetesReporter) logSriovNodeState() {
fmt.Fprintf(r.dumpOutput, "Logging node states")

nodeStates, err := r.clients.SriovNetworkNodeStates("openshift-sriov-network-operator").List(context.Background(), metav1.ListOptions{})
if err != nil {
fmt.Fprintf(os.Stderr, "failed to fetch node states: %v\n", err)
return
err := os.Mkdir(reportPath, 0755)
if err != nil && !errors.Is(err, os.ErrExist) {
return nil, err
}

j, err := json.MarshalIndent(nodeStates, "", " ")
reporter, err := kniK8sReporter.New("", addToScheme, dumpNamespace, reportPath, crds...)
if err != nil {
fmt.Println("Failed to marshal node states")
return
return nil, err
}
fmt.Fprintln(r.dumpOutput, string(j))
return reporter, nil
}
Loading

0 comments on commit 992723c

Please sign in to comment.