chore(test): setup flappy tests

Signed-off-by: Manfred Touron <[email protected]>

Makefile

@@ -55,19 +55,23 @@ examples.build: install_gnodev examples.precompile
 
 ########################################
 # Test suite
-.PHONY: test test.go test.gno test.files1 test.files2 test.realm test.packages
-test: test.gno test.go
+.PHONY: test test.go test.gno test.files1 test.files2 test.realm test.packages test.flappy
+test: test.gno test.go test.flappy
 	@echo "Full test suite finished."
 
 test.gno: test.files1 test.files2 test.realm test.packages test.examples
 	go test tests/*.go -v -run "TestFileStr"
 	go test tests/*.go -v -run "TestSelectors"
 
+test.flappy:
+	# flappy tests should work "sometimes" (at least once)
+	TEST_STABILITY=flappy go run moul.io/testman test -test.v -timeout=20m -retry=10 -run ^TestFlappy ./pkgs/bft/consensus ./pkgs/bft/blockchain
+
 test.go:
 	go test . -v
 	# -p 1 shows test failures as they come
 	# maybe another way to do this?
-	go test ./pkgs/... -v -p 1
+	go test ./pkgs/... -v -p 1 -timeout=20m
 
 test.files1:
 	go test tests/*.go -v -test.short -run "TestFiles1/" --timeout 20m
@@ -104,5 +108,5 @@ genproto2:
 	find pkgs | grep -v "^pkgs\/amino" | grep "pbbindings" | xargs rm
 	find pkgs | grep -v "^pkgs\/amino" | grep "pb.go" | xargs rm
 	#@rm gno.proto || true
-	@rm pbbindings.go || true 
+	@rm pbbindings.go || true
 	@rm gno.pb.go || true

deps.go

@@ -0,0 +1,11 @@
+//go:build tools
+// +build tools
+
+package gno
+
+// this file is used to allow "go mod" to be aware of some development dependencies.
+
+import (
+	// required by Makefile for flappy tests
+	_ "moul.io/testman"
+)

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/btcsuite/btcd v0.20.1-beta
 	github.com/btcsuite/btcutil v1.0.2
 	github.com/davecgh/go-spew v1.1.1
+	github.com/dgraph-io/badger/v3 v3.2103.2
 	github.com/fortytw2/leaktest v1.3.0
 	github.com/gdamore/tcell/v2 v2.1.0
 	github.com/gnolang/cors v1.8.1
@@ -18,6 +19,7 @@ require (
 	github.com/jaekwon/testify v1.6.1
 	github.com/jmhodges/levigo v1.0.0
 	github.com/libp2p/go-buffer-pool v0.0.2
+	github.com/linxGnu/grocksdb v1.7.2
 	github.com/mattn/go-runewidth v0.0.10
 	github.com/pelletier/go-toml v1.9.3
 	github.com/stretchr/testify v1.7.1
@@ -31,26 +33,41 @@ require (
 	golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b
 	golang.org/x/tools v0.1.0
 	google.golang.org/protobuf v1.27.1
+	moul.io/testman v1.5.0
 )
 
 require (
+	github.com/cespare/xxhash v1.1.0 // indirect
+	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/dgraph-io/ristretto v0.1.0 // indirect
+	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
 	github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
 	github.com/gdamore/encoding v1.0.0 // indirect
-	github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
+	github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect
+	github.com/golang/snappy v0.0.3 // indirect
+	github.com/google/flatbuffers v1.12.1 // indirect
 	github.com/gorilla/securecookie v1.1.1 // indirect
 	github.com/gorilla/sessions v1.2.1 // indirect
+	github.com/klauspost/compress v1.12.3 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.0.3 // indirect
+	github.com/peterbourgon/ff/v3 v3.0.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rivo/uniseg v0.1.0 // indirect
+	go.opencensus.io v0.22.5 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect
 	golang.org/x/text v0.3.6 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
-	gopkg.in/yaml.v2 v2.2.2 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99 // indirect
+	moul.io/banner v1.0.1 // indirect
+	moul.io/motd v1.0.0 // indirect
+	moul.io/u v1.6.0 // indirect
 )
 
 replace github.com/gdamore/tcell/v2 => github.com/gnolang/tcell/v2 v2.1.0

pkgs/bft/blockchain/reactor_test.go

@@ -20,6 +20,7 @@ import (
 	"github.com/gnolang/gno/pkgs/errors"
 	"github.com/gnolang/gno/pkgs/log"
 	"github.com/gnolang/gno/pkgs/p2p"
+	"github.com/gnolang/gno/pkgs/testutils"
 )
 
 var config *cfg.Config
@@ -173,7 +174,9 @@ func TestNoBlockResponse(t *testing.T) {
 // or without significant refactoring of the module.
 // Alternatively we could actually dial a TCP conn but
 // that seems extreme.
-func TestBadBlockStopsPeer(t *testing.T) {
+func TestFlappyBadBlockStopsPeer(t *testing.T) {
+	testutils.FilterStability(t, testutils.Flappy)
+
 	config = cfg.ResetTestRoot("blockchain_reactor_test")
 	defer os.RemoveAll(config.RootDir)
 	genDoc, privVals := randGenesisDoc(1, false, 30)

pkgs/bft/consensus/replay_test.go

@@ -32,6 +32,7 @@ import (
 	"github.com/gnolang/gno/pkgs/events"
 	"github.com/gnolang/gno/pkgs/log"
 	"github.com/gnolang/gno/pkgs/random"
+	"github.com/gnolang/gno/pkgs/testutils"
 )
 
 func TestMain(m *testing.M) {
@@ -534,7 +535,9 @@ func TestHandshakeReplayOne(t *testing.T) {
 }
 
 // Sync from caught up
-func TestHandshakeReplayNone(t *testing.T) {
+func TestFlappyHandshakeReplayNone(t *testing.T) {
+	testutils.FilterStability(t, testutils.Flappy)
+
 	for _, m := range modes {
 		testHandshakeReplay(t, config, numBlocks, m, nil)
 	}

pkgs/testutils/flappy.go

@@ -0,0 +1,24 @@
+package testutils
+
+import (
+	"fmt"
+	"os"
+	"testing"
+)
+
+type StabilityMode string
+
+const (
+	Stable StabilityMode = "stable"
+	Flappy StabilityMode = "flappy"
+	Broken StabilityMode = "broken"
+)
+
+func FilterStability(t *testing.T, mode StabilityMode) {
+	t.Helper()
+
+	filter := os.Getenv("STABILITY_FILTER")
+	if filter != string(mode) {
+		t.Skip(fmt.Sprintf("skip test with %q stability", mode))
+	}
+}