build: run four nodes in acceptance tests

This is required for

cockroachdb#17272

and merits a separate review.

Makefile

@@ -31,9 +31,10 @@ TAGS         :=
 TESTS        :=.## Tests to run for use with `make test`.
 BENCHES      :=## Benchmarks to run for use with `make bench`.
 FILES        :=## Space delimited list of logic test files to run, for make testlogic.
-TESTTIMEOUT  := 4m
-RACETIMEOUT  := 15m
-BENCHTIMEOUT := 5m
+TESTTIMEOUT  := 4m## Test timeout to use for regular tests.
+RACETIMEOUT  := 15m## Test timeout to use for race tests.
+ACCEPTANCETIMEOUT := 30m## Test timeout to use for acceptance tests.
+BENCHTIMEOUT := 5m## Test timeout to use for benchmarks.
 TESTFLAGS    :=## Extra flags to pass to the go test runner, e.g. "-v --vmodule=raft=1"
 STRESSFLAGS  :=## Extra flags to pass to `stress` during `make stress`.
 DUPLFLAGS    := -t 100
@@ -268,6 +269,8 @@ upload-coverage: $(BOOTSTRAP_TARGET)
 	@build/upload-coverage.sh
 
 .PHONY: acceptance
+acceptance: TESTTIMEOUT := $(ACCEPTANCETIMEOUT)
+acceptance: export TESTTIMEOUT := $(TESTTIMEOUT)
 acceptance: ## Run acceptance tests.
 	@pkg/acceptance/run.sh
 

build/teamcity-acceptance.sh

@@ -14,4 +14,4 @@ export TMPDIR=$PWD/artifacts/acceptance
 
 build/builder.sh make TYPE=release-linux-gnu testbuild TAGS=acceptance PKG=./pkg/acceptance
 cd pkg/acceptance
-./acceptance.test -nodes 3 -l "$TMPDIR" -test.v -test.timeout 10m 2>&1 | tee "$TMPDIR/acceptance.log" | go-test-teamcity
+./acceptance.test -nodes 4 -l "$TMPDIR" -test.v -test.timeout 10m 2>&1 | tee "$TMPDIR/acceptance.log" | go-test-teamcity

build/teamcity-publish-artifacts.sh

@@ -16,7 +16,7 @@ if [[ "$TC_BUILD_BRANCH" != *alpha* ]] && [ "$TEAMCITY_BUILDCONF_NAME" == 'Publi
 	docker build --tag=$image:{latest,"$TC_BUILD_BRANCH"} build/deploy
 
 	build/builder.sh make TYPE=release-linux-gnu testbuild TAGS=acceptance PKG=./pkg/acceptance
-	(cd pkg/acceptance && ./acceptance.test -i $image -b /cockroach/cockroach -nodes 3 -test.v -test.timeout -5m)
+	(cd pkg/acceptance && ./acceptance.test -i $image -b /cockroach/cockroach -nodes 4 -test.v -test.timeout -5m)
 
 	sed "s/<EMAIL>/$DOCKER_EMAIL/;s/<AUTH>/$DOCKER_AUTH/" < build/.dockercfg.in > ~/.dockercfg
 	docker push "$image:latest"

build/variables.mk

@@ -2,6 +2,7 @@
 # DO NOT EDIT!
 define VALID_VARS
   .DEFAULT_GOAL
+  ACCEPTANCETIMEOUT
   ARCHIVE
   ARCHIVE_BASE
   BENCHES

pkg/acceptance/run.sh

@@ -10,4 +10,4 @@ set -euxo pipefail
 mkdir -p artifacts/acceptance
 export TMPDIR=$PWD/artifacts/acceptance
 
-make test PKG=./pkg/acceptance TAGS=acceptance TESTFLAGS="${TESTFLAGS--v -nodes 3} -l $TMPDIR"
+make test PKG=./pkg/acceptance TESTTIMEOUT="${TESTTIMEOUT-30m}" TAGS=acceptance TESTFLAGS="${TESTFLAGS--v -nodes 4} -l $TMPDIR"

pkg/acceptance/util.go

@@ -84,7 +84,7 @@ func init() {
 }
 
 var flagDuration = flag.Duration("d", cluster.DefaultDuration, "duration to run the test")
-var flagNodes = flag.Int("nodes", 3, "number of nodes")
+var flagNodes = flag.Int("nodes", 4, "number of nodes")
 var flagStores = flag.Int("stores", 1, "number of stores to use for each node")
 var flagRemote = flag.Bool("remote", false, "run the test using terrafarm instead of docker")
 var flagCwd = flag.String("cwd", "../cloud/aws", "directory to run terraform from")