Skip to content

Commit

Permalink
Flake Test fix: Sort TaskRunResults
Browse files Browse the repository at this point in the history
PR tektoncd#7100 introduced a flaky
test. When comparing the pipelineRuns and TaskRuns, the results were not
sorted. As a result, `cmp.Diff` throws an error sometimes when the order
does not match.

This PR sorts the TaskRunResults by providing a sort function to
`cmp.Diff`. After the fix, running this 20 times consecutively,  no flakes were found.
  • Loading branch information
chitrangpatel committed Oct 20, 2023
1 parent 5066b40 commit b4dfa20
Showing 1 changed file with 12 additions and 0 deletions.
12 changes: 12 additions & 0 deletions test/propagated_results_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,16 @@ import (
func TestPropagatedResults(t *testing.T) {
t.Parallel()

// Ignore the Results when comparing the Runs directly. Instead, check the results separately.
// This is because the order of resutls changes and makes the test very flaky.
ignoreTaskRunStatusFields := cmpopts.IgnoreFields(v1.TaskRunStatusFields{}, "Results")
sortTaskRunResults := cmpopts.SortSlices(func(x, y v1.TaskRunResult) bool { return x.Name < y.Name })

ignorePipelineRunStatusFields := cmpopts.IgnoreFields(v1.PipelineRunStatusFields{}, "Provenance")
ignoreTaskRunStatus := cmpopts.IgnoreFields(v1.TaskRunStatusFields{}, "StartTime", "CompletionTime", "Sidecars", "Provenance")
requireAlphaFeatureFlag = requireAnyGate(map[string]string{
"enable-api-fields": "alpha"})

type tests struct {
name string
pipelineName string
Expand Down Expand Up @@ -89,6 +95,7 @@ func TestPropagatedResults(t *testing.T) {
ignoreStepState,
ignoreSAPipelineRunSpec,
ignorePipelineRunStatusFields,
ignoreTaskRunStatusFields,
)
if d != "" {
t.Fatalf(`The resolved spec does not match the expected spec. Here is the diff: %v`, d)
Expand All @@ -105,10 +112,15 @@ func TestPropagatedResults(t *testing.T) {
ignoreContainerStates,
ignoreStepState,
ignoreSATaskRunSpec,
ignoreTaskRunStatusFields,
)
if d != "" {
t.Fatalf(`The expected taskrun does not match created taskrun. Here is the diff: %v`, d)
}
d = cmp.Diff(tr.Status.TaskRunStatusFields.Results, taskrun.Status.TaskRunStatusFields.Results, sortTaskRunResults)
if d != "" {
t.Fatalf(`The expected TaskRunResults does not what was received. Here is the diff: %v`, d)
}
}
t.Logf("Successfully finished test %q", td.name)
})
Expand Down

0 comments on commit b4dfa20

Please sign in to comment.