fix(ENTESB-13151): Add test report generation

- Auto save test results in json format for each test executed to local output dir
- Add new YAKS cli command 'report'
- Add report option to fetch test results from cluster
- Generate report from test results supporting different output formats (junit, json, summary)

.gitignore

@@ -5,6 +5,7 @@
 
 build/_maven_*
 build/_output
+_output
 
 /yaks
 /yaks-*

pkg/apis/yaks/v1alpha1/test_types.go

@@ -74,6 +74,7 @@ type TestList struct {
 type TestResults struct {
 	Summary TestSummary  `json:"summary,omitempty"`
 	Tests	[]TestResult `json:"tests,omitempty"`
+	Errors 	[]string 	 `json:"errors,omitempty"`
 }
 
 type TestSummary struct {
@@ -99,7 +100,7 @@ const (
 	TestKind string = "Test"
 
 	// TestPhaseNone --
-	IntegrationTestPhaseNone TestPhase = ""
+	TestPhaseNone TestPhase = ""
 	// TestPhasePending --
 	TestPhasePending TestPhase = "Pending"
 	// TestPhaseRunning --

pkg/cmd/report.go

@@ -0,0 +1,92 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/citrusframework/yaks/pkg/apis/yaks/v1alpha1"
+	"github.com/citrusframework/yaks/pkg/cmd/report"
+	"github.com/spf13/cobra"
+	ctrl "sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func newCmdReport(rootCmdOptions *RootCmdOptions) *cobra.Command {
+	options := reportCmdOptions{
+		RootCmdOptions: rootCmdOptions,
+	}
+
+	cmd := cobra.Command{
+		PersistentPreRunE: options.preRun,
+		Use:               "report [options]",
+		Short:             "Generate test report from last test run",
+		Long:              `Generate test report from last test run. Test results are fetched from cluster and/or collected from local test output.`,
+		RunE:              options.run,
+		SilenceUsage:      true,
+	}
+
+	cmd.Flags().BoolVar(&options.fetch, "fetch", false, "Fetch latest test results from cluster.")
+	cmd.Flags().VarP(&options.output, "output", "o", "The report output format, one of 'summary', 'json', 'junit'")
+	cmd.Flags().BoolVarP(&options.clean, "clean", "c", false,"Clean the report output folder before fetching results")
+
+	return &cmd
+}
+
+type reportCmdOptions struct {
+	*RootCmdOptions
+	clean bool
+	fetch bool
+	output report.OutputFormat
+}
+
+func (o *reportCmdOptions) run(cmd *cobra.Command, _ []string) error {
+	var results v1alpha1.TestResults
+	if o.fetch {
+		if fetched, err := o.FetchResults(); err == nil {
+			results = *fetched
+		} else {
+			return err
+		}
+	} else if loaded, err := report.LoadTestResults(); err == nil {
+		results = *loaded
+	} else {
+		return err
+	}
+
+	content, err := report.GenerateReport(&results, o.output)
+	if err != nil {
+		return err
+	}
+	_, err = fmt.Fprintf(cmd.OutOrStdout(), "%s\n", content)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (o *reportCmdOptions) FetchResults() (*v1alpha1.TestResults, error) {
+	c, err := o.GetCmdClient()
+	if err != nil {
+		return nil, err;
+	}
+
+	if o.clean {
+		err = report.CleanReports()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	results := v1alpha1.TestResults{}
+	testList := v1alpha1.TestList{}
+	if err := c.List(o.Context, &testList, ctrl.InNamespace(o.Namespace)); err != nil {
+		return nil, err
+	}
+
+	for _, test := range testList.Items {
+		report.AppendTestResults(&results, test.Status.Results)
+		if err := report.SaveTestResults(&test); err != nil {
+			fmt.Printf("Failed to save test results: %s", err.Error())
+		}
+	}
+
+	return &results, nil
+}

pkg/cmd/report/junit.go

@@ -0,0 +1,95 @@
+package report
+
+import (
+	"encoding/xml"
+	"github.com/citrusframework/yaks/pkg/apis/yaks/v1alpha1"
+	"os"
+	"path"
+)
+
+const (
+	XmlProcessingInstruction = `<?xml version="1.0" encoding="UTF-8"?>`
+)
+
+type JUnitReport struct {
+	Suite TestSuite `xml:"testsuite"`
+}
+
+type TestSuite struct {
+	Name string `xml:"name,attr"`
+	Errors int `xml:"errors,attr"`
+	Failures int `xml:"failures,attr"`
+	Skipped int `xml:"skipped,attr"`
+	Tests int `xml:"tests,attr"`
+	Time float32 `xml:"time,attr"`
+	TestCase []TestCase `xml:"testcase"`
+}
+
+type TestCase struct {
+	Name string `xml:"name,attr"`
+	ClassName string `xml:"classname,attr"`
+	Time float32 `xml:"time,attr"`
+	SystemOut string `xml:"system-out,omitempty"`
+	Failure *Failure
+}
+
+type Failure struct {
+	XMLName xml.Name `xml:"failure,omitempty"`
+	Message string `xml:"message,attr,omitempty"`
+	Type string `xml:"type,attr,omitempty"`
+	Stacktrace string `xml:",chardata"`
+}
+
+func createJUnitReport(results *v1alpha1.TestResults, outputDir string) (string, error) {
+	var report = JUnitReport {
+		Suite: TestSuite {
+			Name: "org.citrusframework.yaks.JUnitReport",
+			Failures: results.Summary.Failed,
+			Skipped: results.Summary.Skipped,
+			Tests: results.Summary.Total,
+		},
+	}
+
+	for _, result := range results.Tests {
+		_, testName := path.Split(result.Name)
+		testCase := TestCase{
+			Name: testName,
+			ClassName: result.Name,
+		}
+
+		if len(result.ErrorMessage) > 0 {
+			testCase.Failure = &Failure{
+				Message:    result.ErrorMessage,
+				Type:       result.ErrorType,
+				Stacktrace: "",
+			}
+		}
+
+		report.Suite.TestCase = append(report.Suite.TestCase, testCase)
+	}
+
+	// need to workaround marshalling in order to overwrite local element name of root element
+	tmp := struct {
+		TestSuite
+		XMLName struct{} `xml:"testsuite"`
+	}{TestSuite: report.Suite}
+	if bytes, err := xml.MarshalIndent(tmp,"", "  "); err == nil {
+		junitReport := XmlProcessingInstruction + string(bytes)
+
+		if err := createIfNotExists(outputDir); err != nil {
+			return "", nil
+		}
+
+		reportFile, err := os.Create(path.Join(outputDir, "junit-reports.xml"))
+		if err != nil {
+			return "", err
+		}
+
+		if _, err := reportFile.Write([]byte(junitReport)); err != nil {
+			return "", err
+		}
+		return junitReport, nil
+	} else {
+		return "", err
+	}
+}

pkg/cmd/report/report.go

@@ -0,0 +1,165 @@
+package report
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"github.com/citrusframework/yaks/pkg/apis/yaks/v1alpha1"
+	"github.com/citrusframework/yaks/pkg/util/kubernetes"
+	"io/ioutil"
+	"os"
+	"path"
+)
+
+type OutputFormat string
+
+// Set implements pflag/flag.Value
+func (d *OutputFormat) Set(s string) error {
+	*d = OutputFormat(s)
+	return nil
+}
+
+// Type implements pflag.Value
+func (d *OutputFormat) Type() string {
+	return "string"
+}
+
+// Type implements pflag/flag.Value
+func (d *OutputFormat) String() string {
+	return string(*d)
+}
+
+const (
+	OutputDir = "_output"
+
+	DefaultOutput OutputFormat = ""
+	JsonOutput    OutputFormat = "json"
+	JUnitOutput   OutputFormat = "junit"
+	SummaryOutput OutputFormat = "summary"
+)
+
+func GenerateReport(results *v1alpha1.TestResults, output OutputFormat) (string, error) {
+	switch output {
+		case JUnitOutput:
+			outputDir, err := createInWorkingDir(OutputDir)
+			if err != nil {
+				return "", err
+			}
+
+			if junitReport, err := createJUnitReport(results, outputDir); err != nil {
+				return "", err
+			} else {
+				return junitReport, nil
+			}
+		case SummaryOutput, DefaultOutput:
+			summaryReport := GetSummaryReport(results)
+			return summaryReport, nil
+		case JsonOutput:
+			if bytes, err := json.MarshalIndent(results, "", "  "); err != nil {
+				return "", err
+			} else {
+				return string(bytes), nil
+			}
+		default:
+			return "", errors.New(fmt.Sprintf("Unsupported report output format '%s'. Please use one of 'summary', 'json', 'junit'", output))
+	}
+}
+
+func AppendTestResults(results *v1alpha1.TestResults, result v1alpha1.TestResults) {
+	results.Summary.Passed += result.Summary.Passed
+	results.Summary.Failed += result.Summary.Failed
+	results.Summary.Skipped += result.Summary.Skipped
+	results.Summary.Undefined += result.Summary.Undefined
+	results.Summary.Pending += result.Summary.Pending
+	results.Summary.Total += result.Summary.Total
+
+	for _, result := range result.Tests {
+		results.Tests = append(results.Tests, result)
+	}
+}
+
+func SaveTestResults(test *v1alpha1.Test) error {
+	outputDir, err := createInWorkingDir(OutputDir)
+
+	reportFile, err := os.Create(path.Join(outputDir, kubernetes.SanitizeName(test.Name)) + ".json")
+	if err != nil {
+		return err
+	}
+
+	bytes, _ := json.Marshal(test.Status.Results)
+	if _, err := reportFile.Write(bytes); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func CleanReports() error {
+	err := removeFromWorkingDir(OutputDir)
+	return err
+}
+
+func LoadTestResults() (*v1alpha1.TestResults, error) {
+	results := v1alpha1.TestResults{}
+	outputDir, err := getInWorkingDir(OutputDir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return &results, nil
+		} else {
+			return &results, err
+		}
+	}
+
+	var files []os.FileInfo
+	files, err = ioutil.ReadDir(outputDir)
+	if err != nil {
+		return &results, err
+	}
+
+	for _, file := range files {
+		if path.Ext(file.Name()) != ".json" {
+			continue
+		}
+
+		content, err := ioutil.ReadFile(path.Join(outputDir, file.Name()))
+		if err != nil {
+			return &results, err
+		}
+
+		var result v1alpha1.TestResults
+		err = json.Unmarshal(content, &result)
+		if err != nil {
+			return &results, err
+		}
+
+		AppendTestResults(&results, result)
+	}
+
+	return &results, nil
+}
+
+func PrintSummaryReport(results *v1alpha1.TestResults) {
+	fmt.Printf("%s\n", GetSummaryReport(results))
+}
+
+func GetSummaryReport(results *v1alpha1.TestResults) string {
+	summary := fmt.Sprintf("Test results: Total: %d, Passed: %d, Failed: %d, Skipped: %d\n",
+		results.Summary.Total, results.Summary.Passed, results.Summary.Failed, results.Summary.Skipped)
+
+	for _, test := range results.Tests {
+		result := "Passed"
+		if len(test.ErrorMessage) > 0 {
+			result = fmt.Sprintf("Failure caused by %s - %s", test.ErrorType, test.ErrorMessage)
+		}
+		summary += fmt.Sprintf("\t%s: %s\n", test.Name, result)
+	}
+
+	if len(results.Errors) > 0 {
+		if prettyPrint, err := json.MarshalIndent(results.Errors, "", "  "); err == nil {
+			summary += fmt.Sprintf("\nErrors: \n%s", string(prettyPrint))
+		} else {
+			fmt.Printf("Failed to read error details from test results: %s", err.Error())
+		}
+	}
+	return summary
+}