Add nightly flaky test finder (#1499)

* add CI script for running tests multiple times to find flaky tests

.github/workflows/deflake.yml

@@ -0,0 +1,26 @@
+name: Deflake
+on:
+  schedule:
+    - cron: '22 18 * * 0'
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: 1.17
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+            ~/go/bin/
+            ~/.kubebuilder/bin/k8s
+          key: ${{ runner.os }}-go-cache-${{ hashFiles('**/go.sum') }}
+      - name: Running tests 10 times to find flaky tests
+        run: |
+          make toolchain
+          make deflake
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

Makefile

@@ -26,11 +26,16 @@ ci: toolchain verify licenses battletest ## Run all steps used by continuous int
 test: ## Run tests
 	ginkgo -r
 
-battletest: ## Run stronger tests
+strongertests:
 	# Run randomized, parallelized, racing, code coveraged, tests
 	ginkgo -r \
-		-cover -coverprofile=coverage.out -outputdir=. -coverpkg=./pkg/... \
-		--randomizeAllSpecs --randomizeSuites -race
+			-cover -coverprofile=coverage.out -outputdir=. -coverpkg=./pkg/... \
+			--randomizeAllSpecs --randomizeSuites -race
+
+deflake:
+	for i in {1..10}; do make strongertests || exit 1; done
+
+battletest: strongertests
 	go tool cover -html coverage.out -o coverage.html
 
 verify: codegen ## Verify code. Includes dependencies, linting, formatting, etc