Improve pipeline stress test

- Improve stability by increasing test duration, timeouts and watchdog
  timer
- Add test start/stop messages if run with `-v`. Help with travis timing
  out tests with 10min without any output
- Add all active go-routine stack traces to errors

.travis.yml

@@ -68,7 +68,7 @@ jobs:
       go: $GO_VERSION
       stage: test
     - os: linux
-      env: TARGETS="-C libbeat stress-tests"
+      env: STRESS_TEST_OPTIONS="-timeout=20m -race -v -parallel 1" TARGETS="-C libbeat stress-tests"
       go: $GO_VERSION
       stage: test
 

libbeat/publisher/pipeline/stress/gen.go

@@ -1,7 +1,9 @@
 package stress
 
 import (
+	"bytes"
 	"fmt"
+	"runtime/pprof"
 	"sync"
 	"time"
 
@@ -25,7 +27,7 @@ var defaultGenerateConfig = generateConfig{
 	ACK:       false,
 	MaxEvents: 0,
 	WaitClose: 0,
-	Watchdog:  1 * time.Second,
+	Watchdog:  2 * time.Second,
 }
 
 var publishModes = map[string]beat.PublishMode{
@@ -92,7 +94,7 @@ func generate(
 		// start generator watchdog
 		withWG(&wg, func() {
 			last := uint64(0)
-			ticker := time.NewTicker(config.Watchdog) // todo: make ticker interval configurable
+			ticker := time.NewTicker(config.Watchdog)
 			defer ticker.Stop()
 			for {
 				select {
@@ -105,7 +107,11 @@ func generate(
 
 				current := count.Load()
 				if last == current {
-					err := fmt.Errorf("no progress in generators (last=%v, current=%v)", last, current)
+					// collect all active go-routines stack-traces:
+					var buf bytes.Buffer
+					pprof.Lookup("goroutine").WriteTo(&buf, 2)
+
+					err := fmt.Errorf("no progress in generator %v (last=%v, current=%v):\n%s", id, last, current, buf.Bytes())
 					errors(err)
 				}
 				last = current

libbeat/publisher/pipeline/stress/stress_test.go

@@ -43,12 +43,22 @@ func TestPipeline(t *testing.T) {
 	}
 
 	if duration == 0 {
-		duration = 10 * time.Second
+		duration = 15 * time.Second
 	}
 
 	configTest(t, "gen", genConfigs, func(t *testing.T, gen string) {
 		configTest(t, "pipeline", pipelineConfigs, func(t *testing.T, pipeline string) {
 			configTest(t, "out", outConfigs, func(t *testing.T, out string) {
+
+				if testing.Verbose() {
+					start := time.Now()
+					fmt.Printf("%v Start stress test %v\n", start.Format(time.RFC3339), t.Name())
+					defer func() {
+						end := time.Now()
+						fmt.Printf("%v Finished stress test %v. Duration=%v\n", end.Format(time.RFC3339), t.Name(), end.Sub(start))
+					}()
+				}
+
 				config, err := common.LoadFiles(gen, pipeline, out)
 				if err != nil {
 					t.Fatal(err)

libbeat/scripts/Makefile

@@ -52,6 +52,7 @@ NOSETESTS_OPTIONS?=--process-timeout=$(TIMEOUT) --with-timer -v --with-xunit --x
 TEST_ENVIRONMENT?=false ## @testing if true, "make testsuite" runs integration tests and system tests in a dockerized test environment
 SYSTEM_TESTS?=false ## @testing if true, "make test" and "make testsuite" run unit tests and system tests
 STRESS_TESTS?=false ## @testing if true, "make test" and "make testsuite" run also run the stress tests
+STRESS_TEST_OPTIONS?=-timeout=20m -race -v
 GOX_OS?=linux darwin windows freebsd netbsd openbsd ## @Building List of all OS to be supported by "make crosscompile".
 GOX_OSARCH?=!darwin/arm !darwin/arm64 !darwin/386 ## @building Space separated list of GOOS/GOARCH pairs to build by "make crosscompile".
 GOX_FLAGS?= ## @building Additional flags to append to the gox command used by "make crosscompile".
@@ -206,7 +207,7 @@ fast-system-tests: ${BEAT_NAME}.test python-env
 stress-tests: ## @testing Runs the stress tests with race detector enabled
 stress-tests:
 	if [ -n '${GOPACKAGES_STRESSTESTS}' ]; then \
-		go test -race --tags=stresstest -v ${GOPACKAGES_STRESSTESTS}; \
+		go test --tags=stresstest ${STRESS_TEST_OPTIONS} ${GOPACKAGES_STRESSTESTS}; \
 	fi
 
 # Run benchmark tests