Add filtering to test-report (#2649)

* Use test name filtering by source lane for SSP Signed-off-by: Daniel Hiller <[email protected]> * Add filtering by test name and not run tests Signed-off-by: Daniel Hiller <[email protected]> * Update image version in wrapper script Signed-off-by: Daniel Hiller <[email protected]> --------- Signed-off-by: Daniel Hiller <[email protected]>
kubevirt · Mar 8, 2023 · eccb80f · eccb80f
1 parent b3b0415
commit eccb80f
Show file tree

Hide file tree

Showing 6 changed files with 160 additions and 35 deletions.
diff --git a/hack/test-report.sh b/hack/test-report.sh
@@ -3,7 +3,7 @@
 function run_test_report() {
     podman run -v "$tmp_dir:/tmp:Z" \
             --network host \
-            quay.io/kubevirtci/test-report:v20230130-af3bc0a4 \
+            quay.io/kubevirtci/test-report:v20230303-9f7c25ce \
             "$@"
 }
 

diff --git a/robots/cmd/test-report/cmd/execution/execution.go b/robots/cmd/test-report/cmd/execution/execution.go
@@ -80,7 +80,11 @@ func init() {
 	executionCmd.PersistentFlags().StringVar(&executionReportFlagOpts.endpoint, "endpoint", test_report.DefaultJenkinsBaseUrl, "jenkins base url")
 	executionCmd.PersistentFlags().DurationVar(&executionReportFlagOpts.startFrom, "start-from", 14*24*time.Hour, "time period for report")
 	executionCmd.PersistentFlags().StringVar(&executionReportFlagOpts.configFile, "config-file", "", "yaml file that contains job names associated with dont_run_tests.json and the job name pattern, if set overrides default-config.yaml")
-	executionCmd.PersistentFlags().StringVar(&executionReportFlagOpts.config, "config", "default", "one of {'default', 'compute', 'storage', 'network'}, chooses one of the default configurations, if set overrides default-config.yaml")
+	var keys []string
+	for key := range configs {
+		keys = append(keys, key)
+	}
+	executionCmd.PersistentFlags().StringVar(&executionReportFlagOpts.config, "config", "default", fmt.Sprintf("one of { %s }, chooses one of the default configurations, if set overrides default-config.yaml", strings.Join(keys, ", ")))
 	executionCmd.PersistentFlags().StringVar(&executionReportFlagOpts.outputFile, "outputFile", "", "Path to output file, if not given, a temporary file will be used")
 	executionCmd.PersistentFlags().BoolVar(&executionReportFlagOpts.overwrite, "overwrite", true, "overwrite output file")
 	executionCmd.PersistentFlags().BoolVar(&executionReportFlagOpts.dryRun, "dry-run", false, "only check which jobs would be considered, do not create an actual report")
@@ -239,7 +243,11 @@ func runExecutionReport() error {
 	startOfReport := time.Now().Add(-1 * executionReportFlagOpts.startFrom)
 	endOfReport := time.Now()
 
-	testNamesToJobNamesToExecutionStatus := test_report.GetTestNamesToJobNamesToTestExecutions(jobs, startOfReport, ctx, regexp.MustCompile(config.TestNamePattern))
+	var jobNamePatternForTestNames *regexp.Regexp
+	if config.JobNamePatternForTestNames != "" {
+		jobNamePatternForTestNames = regexp.MustCompile(config.JobNamePatternForTestNames)
+	}
+	testNamesToJobNamesToExecutionStatus := test_report.GetTestNamesToJobNamesToTestExecutions(jobs, startOfReport, ctx, regexp.MustCompile(config.TestNamePattern), jobNamePatternForTestNames)
 
 	err = writeJsonBaseDataFile(testNamesToJobNamesToExecutionStatus)
 	if err != nil {

diff --git a/robots/cmd/test-report/cmd/execution/execution_test.go b/robots/cmd/test-report/cmd/execution/execution_test.go
@@ -46,7 +46,7 @@ func Test_writeHTMLReportToOutput(t *testing.T) {
 			name: "test template",
 			args: args{
 				htmlReportOutputWriter: os.Stdout,
-				testNames:              []string{"a", "b", "c"},
+				testNames:              []string{"a", "b", "c", "d"},
 				filteredTestNames:      []string{"la", "le", "lu"},
 				skippedTests: map[string]interface{}{
 					"a": struct{}{}},
@@ -66,6 +66,11 @@ func Test_writeHTMLReportToOutput(t *testing.T) {
 						"job2": test_report.TestExecution_Skipped,
 						"job3": test_report.TestExecution_Run,
 					},
+					"d": {
+						"job1": test_report.TestExecution_Unsupported,
+						"job2": test_report.TestExecution_Skipped,
+						"job3": test_report.TestExecution_NoData,
+					},
 				},
 				err:  nil,
 				jLog: nil,

diff --git a/robots/cmd/test-report/cmd/execution/ssp-config.yaml b/robots/cmd/test-report/cmd/execution/ssp-config.yaml
@@ -1,4 +1,5 @@
 jobNamePattern: ^test-(ssp-cnv-4.1[1-9]|kubevirt-cnv-4.1[1-9]-quarantined-ocs)$
+jobNamePatternForTestNames: ^test-ssp-cnv-4.1[1-9]$
 jobNamePatternsToDontRunFileURLs:
   - jobNamePattern: .*cnv-4\.13.*
     dontRunFileURL: https://gitlab.cee.redhat.com/contra/cnv-qe-automation/-/raw/master/tests/tier1/kubevirt/dont_run_tests.json
@@ -9,4 +10,4 @@ jobNamePatternsToDontRunFileURLs:
   - jobNamePattern: .*cnv-4\.10.*
     dontRunFileURL: https://gitlab.cee.redhat.com/contra/cnv-qe-automation/-/raw/cnv-4.10/tests/tier1/kubevirt/dont_run_tests.json
 maxConnsPerHost: 3
-testNamePattern: ^(Common templates|Crypto Policy|DataSources|Metrics|Node Labeller|Observed generation|Prometheus Alerts|RHEL VM|SCC annotation|SSPOperatorReconcilingProperly|Service Controller|Single Node Topology|Template validator|Upgrade from|Validation webhook)
+testNamePattern: .*
diff --git a/robots/cmd/test-report/cmd/execution/test-report.gohtml b/robots/cmd/test-report/cmd/execution/test-report.gohtml
@@ -1,43 +1,28 @@
 <html>
 <head>
     <title>test execution report</title>
-    <meta charset="UTF-8">
+    <meta charset="UTF-8"/>
     <style>
         table, th, td {
             border: 1px solid black;
         }
+        table.noborder, th.noborder, td.noborder {
+            border: 0px;
+        }
         .yellow {
             background-color: #ffff80;
         }
-        .almostgreen {
-            background-color: #dfff80;
-        }
         .green {
             background-color: #9fff80;
         }
         .red {
             background-color: #ff8080;
         }
-        .orange {
-            background-color: #ffbf80;
-        }
         .gray {
             background-color: #898989;
         }
         .unimportant {
         }
-        .tests_passed {
-            color: #226c18;
-            font-weight: bold;
-        }
-        .tests_failed {
-            color: #8a1717;
-            font-weight: bold;
-        }
-        .tests_skipped {
-            color: #535453;
-            font-weight: bold;
-        }
         .right {
             text-align: right;
             width: 100%;
@@ -82,14 +67,74 @@
         }
 
     </style>
-    </meta>
+    <script>
+        function enableFilterFields() {
+            document.getElementById("filterByName").disabled = false;
+            document.getElementById("excludeByName").disabled = false;
+            document.getElementById("filterByNotRunCheckBox").disabled = false;
+        }
+        function updateFilteredRows() {
+            let filter, table, tr, td, i, txtValue, checked, shouldShow, rowsShown;
+            filterTerms = document.getElementById("filterByName").value.toUpperCase().split("|");
+            excludeTerms = document.getElementById("excludeByName").value.toUpperCase().split("|");
+            checked = document.getElementById("filterByNotRunCheckBox").checked;
+            table = document.getElementById("report");
+            tr = table.getElementsByTagName("tr");
+
+            rowsShown = 0;
+            for (i = 1; i < tr.length; i++) {
+                td = tr[i].getElementsByTagName("td")[1];
+                if (td) {
+                    shouldShow = true
+                    txtValue = td.textContent || td.innerText;
+                    if (checked && td.className !== "red") {
+                        shouldShow = false
+                    }
+                    if (excludeTerms.length > 0 && excludeTerms[0] !== "") {
+                        for (k = 0; k < excludeTerms.length; k++) {
+                            if (txtValue.toUpperCase().indexOf(excludeTerms[k]) !== -1) {
+                                shouldShow = false
+                                break;
+                            }
+                        }
+                    }
+                    if (shouldShow === true && filterTerms.length > 0 && filterTerms[0] !== "") {
+                        let found = false
+                        for (k = 0; k < filterTerms.length; k++) {
+                            if (txtValue.toUpperCase().indexOf(filterTerms[k]) !== -1) {
+                                found = true
+                                break;
+                            }
+                        }
+                        if (found !== true) {
+                            shouldShow = false
+                        }
+                    }
+                    if (shouldShow === true) {
+                        tr[i].style.display = "";
+                        rowsShown++;
+                    } else {
+                        tr[i].style.display = "none";
+                    }
+                }
+            }
+            updateRowsShown(rowsShown);
+        }
+        function initRowsShown() {
+            updateRowsShown(document.getElementById("report").getElementsByTagName("tr").length - 1);
+        }
+        function updateRowsShown(rowsShown) {
+            let rowsTotal = document.getElementById("report").getElementsByTagName("tr").length - 1;
+            document.getElementById("totalRowsShown").innerText = "Showing "+rowsShown+" of "+rowsTotal;
+        }
+    </script>
 </head>
-<body>
+<body onload="initRowsShown();enableFilterFields();">
 {{- /* gotype: kubevirt.io/project-infra/robots/main.Data */ -}}
 <h1>test execution report</h1>
 <div>
     {{ $.ReportConfigName }} report configuration<br/>
-    data from {{ $.StartOfReport }} till {{ $.EndOfReport }}
+    data from {{ $.StartOfReport }} till {{ $.EndOfReport }}<br/>
 </div>
 
 <div id="reportConfig" class="popup right" >
@@ -115,12 +160,47 @@
     </div>
 </div>
 
-<table>
+<table class="noborder">
+    <tr>
+        <td class="noborder">
+        <b>Filters:</b>
+        </td>
+        <td class="noborder">
+            <label for="filterByName">Include tests that contain</label>
+        </td>
+        <td class="noborder">
+            <input type="text" id="filterByName" onkeyup="updateFilteredRows()" placeholder="term1|term2|..." disabled>
+        </td>
+    </tr>
+    <tr>
+        <td class="noborder">
+        </td>
+        <td class="noborder">
+            <label for="excludeByName">Exclude tests that contain</label>
+        </td>
+        <td class="noborder">
+            <input type="text" id="excludeByName" onkeyup="updateFilteredRows()" placeholder="term1|term2|..." disabled>
+        </td>
+    </tr>
+    <tr>
+        <td class="noborder">
+        </td>
+        <td class="noborder">
+    <label for="filterByNotRunCheckBox">Only not run tests</label>
+        </td>
+        <td class="noborder">
+            <input type="checkbox" id="filterByNotRunCheckBox" onClick="updateFilteredRows()" title="Show only rows that have not been run" disabled>
+        </td>
+    </tr>
+</table>
+
+<div id="totalRowsShown"><i>Loading rows...</i></div>
+<table id="report">
     <tr>
-        <td></td>
-        <td></td>
+        <th></th>
+        <th></th>
         {{ range $job := $.LookedAtJobs }}
-            <td><a href="{{ $.JenkinsBaseURL }}/job/{{ $job }}/">{{ $job }}</a></td>
+            <th><a href="{{ $.JenkinsBaseURL }}/job/{{ $job }}/">{{ $job }}</a></th>
         {{ end }}
     </tr>
     {{ range $row, $test := $.TestNames }}
@@ -129,14 +209,25 @@
             <td class="{{ if (index $.SkippedTests $test) }}red{{ end }}">{{ $test }}</td>
             {{ range $col, $job := $.LookedAtJobs }}
                 <td class="center">{{ with $skipped := (index $.TestNamesToJobNamesToSkipped $test $job) }}
-                        <div id="r{{$row}}c{{$col}}" title="test {{ if eq $skipped (index $.TestExecutionMapping "TestExecution_Skipped") }}skipped{{ else if eq $skipped (index $.TestExecutionMapping "TestExecution_Run") }}run{{ else if eq $skipped (index $.TestExecutionMapping "TestExecution_Unsupported") }}unsupported{{ else }}{{ end }}" class="{{ if eq $skipped (index $.TestExecutionMapping "TestExecution_Skipped") }}yellow{{ else if eq $skipped (index $.TestExecutionMapping "TestExecution_Run") }}green{{ else if eq $skipped (index $.TestExecutionMapping "TestExecution_Unsupported") }}gray{{ else }}{{ end }}" >
-                            <input title="{{ $test }} &#013; {{ $job }}" type="checkbox" readonly {{ if eq $skipped (index $.TestExecutionMapping "TestExecution_Run") }}checked{{ end }}/>
+                        <div id="r{{$row}}c{{$col}}" title="test
+                                {{- if eq $skipped (index $.TestExecutionMapping "TestExecution_Skipped") }} skipped
+                                {{- else if eq $skipped (index $.TestExecutionMapping "TestExecution_Run") }} run
+                                {{- else if eq $skipped (index $.TestExecutionMapping "TestExecution_Unsupported") }} unsupported
+                                {{- else }}
+                                {{ end -}}"
+                             class="
+                                {{- if eq $skipped (index $.TestExecutionMapping "TestExecution_Skipped") }}yellow
+                                {{- else if eq $skipped (index $.TestExecutionMapping "TestExecution_Run") }}green
+                                {{- else if eq $skipped (index $.TestExecutionMapping "TestExecution_Unsupported") }}gray
+                                {{- else }}
+                                {{ end -}}" >
+                            <input title="{{ $test }} &#013; {{ $job }}" type="checkbox" disabled
+                                   {{- if eq $skipped (index $.TestExecutionMapping "TestExecution_Run") }} checked{{ end -}}/>
                         </div>
                     {{ else }}n/a{{ end }}</td>
             {{ end }}
         </tr>
     {{ end }}
 </table>
-
 </body>
 </html>
diff --git a/robots/pkg/test-report/test_report.go b/robots/pkg/test-report/test_report.go
@@ -118,6 +118,10 @@ type Config struct {
 	// JobNamePattern is a regexp.Regexp that describes which jobs are considered for the report
 	JobNamePattern string `yaml:"jobNamePattern"`
 
+	// JobNamePatternForTestNames is a regexp.Regexp that describes which jobs are considered to contain all test names
+	// we should be looking at
+	JobNamePatternForTestNames string `yaml:"jobNamePatternForTestNames,omitempty"`
+
 	// TestNamePattern is a regexp.Regexp that describes what tests are considered for the report
 	TestNamePattern string `yaml:"testNamePattern"`
 
@@ -206,7 +210,7 @@ func CreateReportData(jobNamePatternsToTestNameFilterRegexps map[*regexp.Regexp]
 	return data
 }
 
-func GetTestNamesToJobNamesToTestExecutions(jobs []*gojenkins.Job, startOfReport time.Time, ctx context.Context, testNamePattern *regexp.Regexp) map[string]map[string]int {
+func GetTestNamesToJobNamesToTestExecutions(jobs []*gojenkins.Job, startOfReport time.Time, ctx context.Context, testNamePattern *regexp.Regexp, jobNamePatternForTestNames *regexp.Regexp) map[string]map[string]int {
 	resultsChan := make(chan map[string]map[string]int)
 	go getTestNamesToJobNamesToTestExecutionForAllJobs(resultsChan, jobs, startOfReport, ctx, logger)
 
@@ -226,6 +230,22 @@ func GetTestNamesToJobNamesToTestExecutions(jobs []*gojenkins.Job, startOfReport
 			}
 		}
 	}
+
+	if jobNamePatternForTestNames != nil {
+		for testName, jobNamesToExecutionStatus := range testNamesToJobNamesToExecutionStatus {
+			jobNamePatternMatchesAnyJobNameForTestNames := false
+			for jobName := range jobNamesToExecutionStatus {
+				if jobNamePatternForTestNames.MatchString(jobName) {
+					jobNamePatternMatchesAnyJobNameForTestNames = true
+					break
+				}
+			}
+			if !jobNamePatternMatchesAnyJobNameForTestNames {
+				delete(testNamesToJobNamesToExecutionStatus, testName)
+			}
+		}
+	}
+
 	return testNamesToJobNamesToExecutionStatus
 }