From e812954bd97bdd2697acfc1d626c55896b95b6ab Mon Sep 17 00:00:00 2001
From: Mahmood Ali <mahmood@hashicorp.com>
Date: Mon, 2 Mar 2020 10:29:24 -0500
Subject: [PATCH 1/3] Simplify Bootstrap logic in tests

This change updates tests to honor `BootstrapExpect` exclusively when
forming test clusters and removes test only knobs, e.g.
`config.DevDisableBootstrap`.

Background:

Test cluster creation is fragile.  Test servers don't follow the
BootstapExpected route like production clusters.  Instead they start as
single node clusters and then get rejoin and may risk causing brain
split or other test flakiness.

The test framework expose few knobs to control those (e.g.
`config.DevDisableBootstrap` and `config.Bootstrap`) that control
whether a server should bootstrap the cluster.  These flags are
confusing and it's unclear when to use: their usage in multi-node
cluster isn't properly documented.  Furthermore, they have some bad
side-effects as they don't control Raft library: If
`config.DevDisableBootstrap` is true, the test server may not
immediately attempt to bootstrap a cluster, but after an election
timeout (~50ms), Raft may force a leadership election and win it (with
only one vote) and cause a split brain.

The knobs are also confusing as Bootstrap is an overloaded term.  In
BootstrapExpect, we refer to bootstrapping the cluster only after N
servers are connected.  But in tests and the knobs above, it refers to
whether the server is a single node cluster and shouldn't wait for any
other server.

Changes:

This commit makes two changes:

First, it relies on `BootstrapExpected` instead of `Bootstrap` and/or
`DevMode` flags.  This change is relatively trivial.

Introduce a `Bootstrapped` flag to track if the cluster is bootstrapped.
This allows us to keep `BootstrapExpected` immutable.  Previously, the
flag was a config value but it gets set to 0 after cluster bootstrap
completes.
---
 client/rpc_test.go                  |  1 -
 command/agent/agent.go              |  7 +--
 command/agent/agent_test.go         |  6 +--
 command/agent/config.go             |  3 +-
 command/agent/testagent.go          |  6 +--
 nomad/acl_endpoint_test.go          |  2 -
 nomad/autopilot_test.go             | 33 ++++++------
 nomad/client_agent_endpoint_test.go | 20 +++++---
 nomad/client_alloc_endpoint_test.go | 30 +++++++----
 nomad/client_fs_endpoint_test.go    | 24 ++++++---
 nomad/client_rpc_test.go            | 36 ++++++++-----
 nomad/client_stats_endpoint_test.go |  6 ++-
 nomad/config.go                     | 25 +++++----
 nomad/heartbeat_test.go             |  9 ++--
 nomad/leader_test.go                | 80 ++++++++++++++++-------------
 nomad/node_endpoint_test.go         | 10 ++--
 nomad/rpc_test.go                   | 18 +++----
 nomad/serf.go                       | 13 ++---
 nomad/serf_test.go                  | 15 +-----
 nomad/server.go                     | 18 ++++---
 nomad/server_test.go                |  8 ---
 nomad/stats_fetcher_test.go         |  1 -
 nomad/testing.go                    |  1 +
 23 files changed, 193 insertions(+), 179 deletions(-)

diff --git a/client/rpc_test.go b/client/rpc_test.go
index 943ca6f1e61..2f8c1337874 100644
--- a/client/rpc_test.go
+++ b/client/rpc_test.go
@@ -63,7 +63,6 @@ func TestRpc_streamingRpcConn_badEndpoint_TLS(t *testing.T) {
 	s1, cleanupS1 := nomad.TestServer(t, func(c *nomad.Config) {
 		c.Region = "regionFoo"
 		c.BootstrapExpect = 1
-		c.DevDisableBootstrap = true
 		c.TLSConfig = &sconfig.TLSConfig{
 			EnableHTTP:           true,
 			EnableRPC:            true,
diff --git a/command/agent/agent.go b/command/agent/agent.go
index c918b03e797..04eff1cc667 100644
--- a/command/agent/agent.go
+++ b/command/agent/agent.go
@@ -11,7 +11,6 @@ import (
 	"runtime"
 	"strings"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	metrics "github.com/armon/go-metrics"
@@ -157,11 +156,7 @@ func convertServerConfig(agentConfig *Config) (*nomad.Config, error) {
 		conf.NodeName = agentConfig.NodeName
 	}
 	if agentConfig.Server.BootstrapExpect > 0 {
-		if agentConfig.Server.BootstrapExpect == 1 {
-			conf.Bootstrap = true
-		} else {
-			atomic.StoreInt32(&conf.BootstrapExpect, int32(agentConfig.Server.BootstrapExpect))
-		}
+		conf.BootstrapExpect = agentConfig.Server.BootstrapExpect
 	}
 	if agentConfig.DataDir != "" {
 		conf.DataDir = filepath.Join(agentConfig.DataDir, "server")
diff --git a/command/agent/agent_test.go b/command/agent/agent_test.go
index 09a5cde4e3d..9dcd02a98b8 100644
--- a/command/agent/agent_test.go
+++ b/command/agent/agent_test.go
@@ -166,14 +166,12 @@ func TestAgent_ServerConfig(t *testing.T) {
 	conf.Server.BootstrapExpect = 1
 	out, err = a.serverConfig()
 	require.NoError(t, err)
-	require.True(t, out.Bootstrap)
-	require.Equal(t, int32(0), out.BootstrapExpect)
+	require.Equal(t, 1, out.BootstrapExpect)
 
 	conf.Server.BootstrapExpect = 3
 	out, err = a.serverConfig()
 	require.NoError(t, err)
-	require.False(t, out.Bootstrap)
-	require.Equal(t, int32(3), out.BootstrapExpect)
+	require.Equal(t, 3, out.BootstrapExpect)
 }
 
 func TestAgent_ServerConfig_SchedulerFlags(t *testing.T) {
diff --git a/command/agent/config.go b/command/agent/config.go
index 2e560ac423d..2df63294c16 100644
--- a/command/agent/config.go
+++ b/command/agent/config.go
@@ -778,7 +778,8 @@ func DevConfig(mode *devModeConfig) *Config {
 	conf.LogLevel = "DEBUG"
 	conf.Client.Enabled = true
 	conf.Server.Enabled = true
-	conf.DevMode = mode != nil
+	conf.DevMode = true
+	conf.Server.BootstrapExpect = 1
 	conf.EnableDebug = true
 	conf.DisableAnonymousSignature = true
 	conf.Consul.AutoAdvertise = helper.BoolToPtr(true)
diff --git a/command/agent/testagent.go b/command/agent/testagent.go
index b2e317574c6..64fb3193db5 100644
--- a/command/agent/testagent.go
+++ b/command/agent/testagent.go
@@ -168,7 +168,7 @@ RETRY:
 	}
 
 	failed := false
-	if a.Config.NomadConfig.Bootstrap && a.Config.Server.Enabled {
+	if a.Config.NomadConfig.BootstrapExpect == 1 && a.Config.Server.Enabled {
 		testutil.WaitForResult(func() (bool, error) {
 			args := &structs.GenericRequest{}
 			var leader string
@@ -358,10 +358,6 @@ func (a *TestAgent) config() *Config {
 	config.ServerHealthInterval = 50 * time.Millisecond
 	config.AutopilotInterval = 100 * time.Millisecond
 
-	// Bootstrap ourselves
-	config.Bootstrap = true
-	config.BootstrapExpect = 1
-
 	// Tighten the fingerprinter timeouts
 	if conf.Client.Options == nil {
 		conf.Client.Options = make(map[string]string)
diff --git a/nomad/acl_endpoint_test.go b/nomad/acl_endpoint_test.go
index 1d83d13b464..7cf5c63998f 100644
--- a/nomad/acl_endpoint_test.go
+++ b/nomad/acl_endpoint_test.go
@@ -1077,8 +1077,6 @@ func TestACLEndpoint_Bootstrap_Reset(t *testing.T) {
 		c.ACLEnabled = true
 		c.DataDir = dir
 		c.DevMode = false
-		c.Bootstrap = true
-		c.DevDisableBootstrap = false
 	})
 	defer cleanupS1()
 	codec := rpcClient(t, s1)
diff --git a/nomad/autopilot_test.go b/nomad/autopilot_test.go
index c46a9cb5159..bd2715b4690 100644
--- a/nomad/autopilot_test.go
+++ b/nomad/autopilot_test.go
@@ -73,7 +73,6 @@ func TestAutopilot_CleanupDeadServer(t *testing.T) {
 
 func testCleanupDeadServer(t *testing.T, raftVersion int) {
 	conf := func(c *Config) {
-		c.DevDisableBootstrap = true
 		c.BootstrapExpect = 3
 		c.RaftConfig.ProtocolVersion = raft.ProtocolVersion(raftVersion)
 	}
@@ -127,13 +126,13 @@ func testCleanupDeadServer(t *testing.T, raftVersion int) {
 func TestAutopilot_CleanupDeadServerPeriodic(t *testing.T) {
 	t.Parallel()
 
-	s1, cleanupS1 := TestServer(t, nil)
-	defer cleanupS1()
-
 	conf := func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 5
 	}
 
+	s1, cleanupS1 := TestServer(t, conf)
+	defer cleanupS1()
+
 	s2, cleanupS2 := TestServer(t, conf)
 	defer cleanupS2()
 
@@ -174,16 +173,14 @@ func TestAutopilot_CleanupDeadServerPeriodic(t *testing.T) {
 func TestAutopilot_RollingUpdate(t *testing.T) {
 	t.Parallel()
 
-	s1, cleanupS1 := TestServer(t, func(c *Config) {
-		c.RaftConfig.ProtocolVersion = 3
-	})
-	defer cleanupS1()
-
 	conf := func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 3
 		c.RaftConfig.ProtocolVersion = 3
 	}
 
+	s1, cleanupS1 := TestServer(t, conf)
+	defer cleanupS1()
+
 	s2, cleanupS2 := TestServer(t, conf)
 	defer cleanupS2()
 
@@ -248,19 +245,21 @@ func TestAutopilot_CleanupStaleRaftServer(t *testing.T) {
 	t.Skip("TestAutopilot_CleanupDeadServer is very flaky, removing it for now")
 	t.Parallel()
 
-	s1, cleanupS1 := TestServer(t, nil)
-	defer cleanupS1()
-
 	conf := func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 3
 	}
+	s1, cleanupS1 := TestServer(t, conf)
+	defer cleanupS1()
+
 	s2, cleanupS2 := TestServer(t, conf)
 	defer cleanupS2()
 
 	s3, cleanupS3 := TestServer(t, conf)
 	defer cleanupS3()
 
-	s4, cleanupS4 := TestServer(t, conf)
+	s4, cleanupS4 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 0
+	})
 	defer cleanupS4()
 
 	servers := []*Server{s1, s2, s3}
@@ -304,7 +303,7 @@ func TestAutopilot_PromoteNonVoter(t *testing.T) {
 	testutil.WaitForLeader(t, s1.RPC)
 
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 0
 		c.RaftConfig.ProtocolVersion = 3
 	})
 	defer cleanupS2()
diff --git a/nomad/client_agent_endpoint_test.go b/nomad/client_agent_endpoint_test.go
index c87cf51b2d9..bc295d00a0b 100644
--- a/nomad/client_agent_endpoint_test.go
+++ b/nomad/client_agent_endpoint_test.go
@@ -30,10 +30,12 @@ func TestMonitor_Monitor_Remote_Client(t *testing.T) {
 	require := require.New(t)
 
 	// start server and client
-	s1, cleanupS1 := TestServer(t, nil)
+	s1, cleanupS1 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 2
+	})
 	defer cleanupS1()
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 2
 	})
 	defer cleanupS2()
 	TestJoin(t, s1, s2)
@@ -125,15 +127,16 @@ func TestMonitor_Monitor_RemoteServer(t *testing.T) {
 	foreignRegion := "foo"
 
 	// start servers
-	s1, cleanupS1 := TestServer(t, nil)
+	s1, cleanupS1 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 2
+	})
 	defer cleanupS1()
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 2
 	})
 	defer cleanupS2()
 
 	s3, cleanupS3 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
 		c.Region = foreignRegion
 	})
 	defer cleanupS3()
@@ -516,12 +519,12 @@ func TestAgentProfile_RemoteClient(t *testing.T) {
 
 	// start server and client
 	s1, cleanup := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 2
 	})
 	defer cleanup()
 
 	s2, cleanup := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 2
 	})
 	defer cleanup()
 
@@ -640,12 +643,13 @@ func TestAgentProfile_Server(t *testing.T) {
 
 	// start servers
 	s1, cleanup := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 2
 		c.EnableDebug = true
 	})
 	defer cleanup()
 
 	s2, cleanup := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 2
 		c.EnableDebug = true
 	})
 	defer cleanup()
diff --git a/nomad/client_alloc_endpoint_test.go b/nomad/client_alloc_endpoint_test.go
index 08a9f148b16..49e665a23c0 100644
--- a/nomad/client_alloc_endpoint_test.go
+++ b/nomad/client_alloc_endpoint_test.go
@@ -186,10 +186,12 @@ func TestClientAllocations_GarbageCollectAll_Remote(t *testing.T) {
 	require := require.New(t)
 
 	// Start a server and client
-	s1, cleanupS1 := TestServer(t, nil)
+	s1, cleanupS1 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 2
+	})
 	defer cleanupS1()
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 2
 	})
 	defer cleanupS2()
 	TestJoin(t, s1, s2)
@@ -432,10 +434,12 @@ func TestClientAllocations_GarbageCollect_Remote(t *testing.T) {
 	require := require.New(t)
 
 	// Start a server and client
-	s1, cleanupS1 := TestServer(t, nil)
+	s1, cleanupS1 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 2
+	})
 	defer cleanupS1()
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 2
 	})
 	defer cleanupS2()
 	TestJoin(t, s1, s2)
@@ -720,10 +724,12 @@ func TestClientAllocations_Stats_Remote(t *testing.T) {
 	require := require.New(t)
 
 	// Start a server and client
-	s1, cleanupS1 := TestServer(t, nil)
+	s1, cleanupS1 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 2
+	})
 	defer cleanupS1()
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 2
 	})
 	defer cleanupS2()
 	TestJoin(t, s1, s2)
@@ -915,10 +921,12 @@ func TestClientAllocations_Restart_Remote(t *testing.T) {
 	require := require.New(t)
 
 	// Start a server and client
-	s1, cleanupS1 := TestServer(t, nil)
+	s1, cleanupS1 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 2
+	})
 	defer cleanupS1()
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 2
 	})
 	defer cleanupS2()
 	TestJoin(t, s1, s2)
@@ -1071,11 +1079,13 @@ func TestAlloc_ExecStreaming(t *testing.T) {
 	t.Parallel()
 
 	////// Nomad clusters topology - not specific to test
-	localServer, cleanupLS := TestServer(t, nil)
+	localServer, cleanupLS := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 2
+	})
 	defer cleanupLS()
 
 	remoteServer, cleanupRS := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 2
 	})
 	defer cleanupRS()
 
diff --git a/nomad/client_fs_endpoint_test.go b/nomad/client_fs_endpoint_test.go
index 821e8abfa21..d07741db4f7 100644
--- a/nomad/client_fs_endpoint_test.go
+++ b/nomad/client_fs_endpoint_test.go
@@ -177,10 +177,12 @@ func TestClientFS_List_Remote(t *testing.T) {
 	require := require.New(t)
 
 	// Start a server and client
-	s1, cleanupS1 := TestServer(t, nil)
+	s1, cleanupS1 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 2
+	})
 	defer cleanupS1()
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 2
 	})
 	defer cleanupS2()
 	TestJoin(t, s1, s2)
@@ -451,10 +453,12 @@ func TestClientFS_Stat_Remote(t *testing.T) {
 	require := require.New(t)
 
 	// Start a server and client
-	s1, cleanupS1 := TestServer(t, nil)
+	s1, cleanupS1 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 2
+	})
 	defer cleanupS1()
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 2
 	})
 	defer cleanupS2()
 	TestJoin(t, s1, s2)
@@ -1000,10 +1004,12 @@ func TestClientFS_Streaming_Remote_Server(t *testing.T) {
 	require := require.New(t)
 
 	// Start a server and client
-	s1, cleanupS1 := TestServer(t, nil)
+	s1, cleanupS1 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 2
+	})
 	defer cleanupS1()
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 2
 	})
 	defer cleanupS2()
 	TestJoin(t, s1, s2)
@@ -1829,10 +1835,12 @@ func TestClientFS_Logs_Remote_Server(t *testing.T) {
 	require := require.New(t)
 
 	// Start a server and client
-	s1, cleanupS1 := TestServer(t, nil)
+	s1, cleanupS1 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 2
+	})
 	defer cleanupS1()
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 2
 	})
 	defer cleanupS2()
 	TestJoin(t, s1, s2)
diff --git a/nomad/client_rpc_test.go b/nomad/client_rpc_test.go
index 6d04805793c..a653440280e 100644
--- a/nomad/client_rpc_test.go
+++ b/nomad/client_rpc_test.go
@@ -88,10 +88,12 @@ func TestServerWithNodeConn_NoPath(t *testing.T) {
 	t.Parallel()
 	require := require.New(t)
 
-	s1, cleanupS1 := TestServer(t, nil)
+	s1, cleanupS1 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 2
+	})
 	defer cleanupS1()
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 2
 	})
 	defer cleanupS2()
 	TestJoin(t, s1, s2)
@@ -122,10 +124,12 @@ func TestServerWithNodeConn_Path(t *testing.T) {
 	t.Parallel()
 	require := require.New(t)
 
-	s1, cleanupS1 := TestServer(t, nil)
+	s1, cleanupS1 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 2
+	})
 	defer cleanupS1()
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 2
 	})
 	defer cleanupS2()
 	TestJoin(t, s1, s2)
@@ -174,14 +178,16 @@ func TestServerWithNodeConn_Path_Newest(t *testing.T) {
 	t.Parallel()
 	require := require.New(t)
 
-	s1, cleanupS1 := TestServer(t, nil)
+	s1, cleanupS1 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 3
+	})
 	defer cleanupS1()
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 3
 	})
 	defer cleanupS2()
 	s3, cleanupS3 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 3
 	})
 	defer cleanupS3()
 	TestJoin(t, s1, s2, s3)
@@ -208,14 +214,16 @@ func TestServerWithNodeConn_PathAndErr(t *testing.T) {
 	t.Parallel()
 	require := require.New(t)
 
-	s1, cleanupS1 := TestServer(t, nil)
+	s1, cleanupS1 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 3
+	})
 	defer cleanupS1()
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 3
 	})
 	defer cleanupS2()
 	s3, cleanupS3 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 3
 	})
 	defer cleanupS3()
 	TestJoin(t, s1, s2, s3)
@@ -242,14 +250,16 @@ func TestServerWithNodeConn_NoPathAndErr(t *testing.T) {
 	t.Parallel()
 	require := require.New(t)
 
-	s1, cleanupS1 := TestServer(t, nil)
+	s1, cleanupS1 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 3
+	})
 	defer cleanupS1()
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 3
 	})
 	defer cleanupS2()
 	s3, cleanupS3 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 3
 	})
 	defer cleanupS3()
 	TestJoin(t, s1, s2, s3)
diff --git a/nomad/client_stats_endpoint_test.go b/nomad/client_stats_endpoint_test.go
index 6e37ef5ba76..d27b418b93f 100644
--- a/nomad/client_stats_endpoint_test.go
+++ b/nomad/client_stats_endpoint_test.go
@@ -173,10 +173,12 @@ func TestClientStats_Stats_Remote(t *testing.T) {
 	require := require.New(t)
 
 	// Start a server and client
-	s1, cleanupS1 := TestServer(t, nil)
+	s1, cleanupS1 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 2
+	})
 	defer cleanupS1()
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 2
 	})
 	defer cleanupS2()
 	TestJoin(t, s1, s2)
diff --git a/nomad/config.go b/nomad/config.go
index 96d6adfa63a..c15268afd98 100644
--- a/nomad/config.go
+++ b/nomad/config.go
@@ -49,16 +49,23 @@ func DefaultRPCAddr() *net.TCPAddr {
 
 // Config is used to parameterize the server
 type Config struct {
-	// Bootstrap mode is used to bring up the first Nomad server.  It is
-	// required so that it can elect a leader without any other nodes
-	// being present
-	Bootstrap bool
+	// Bootstrapped indictes if Server has bootstrapped or not.
+	// Its value must be 0 (not bootstrapped) or 1 (bootstrapped).
+	// All operations on Bootstrapped must be handled via `atomic.*Int32()` calls
+	Bootstrapped int32
 
 	// BootstrapExpect mode is used to automatically bring up a
 	// collection of Nomad servers. This can be used to automatically
-	// bring up a collection of nodes.  All operations on BootstrapExpect
-	// must be handled via `atomic.*Int32()` calls.
-	BootstrapExpect int32
+	// bring up a collection of nodes.
+	//
+	// The BootstrapExpect can be of any of the following values:
+	//  1: Server will form a single node cluster and become a leader immediately
+	//  N, larger than 1: Server will wait until it's connected to N servers
+	//      before attempting leadership and forming the cluster.  No Raft Log operation
+	//      will succeed until then.
+	//  0: Server will wait to get a Raft configuration from another node and may not
+	//      attempt to form a cluster or establish leadership on its own.
+	BootstrapExpect int
 
 	// DataDir is the directory to store our state in
 	DataDir string
@@ -71,10 +78,6 @@ type Config struct {
 	// in the absence of ACLs
 	EnableDebug bool
 
-	// DevDisableBootstrap is used to disable bootstrap mode while
-	// in DevMode. This is largely used for testing.
-	DevDisableBootstrap bool
-
 	// LogOutput is the location to write logs to. If this is not set,
 	// logs will go to stderr.
 	LogOutput io.Writer
diff --git a/nomad/heartbeat_test.go b/nomad/heartbeat_test.go
index f687a4ab485..02dbf79a162 100644
--- a/nomad/heartbeat_test.go
+++ b/nomad/heartbeat_test.go
@@ -69,7 +69,6 @@ func TestHeartbeat_ResetHeartbeatTimer_Nonleader(t *testing.T) {
 
 	s1, cleanupS1 := TestServer(t, func(c *Config) {
 		c.BootstrapExpect = 3 // Won't become leader
-		c.DevDisableBootstrap = true
 	})
 	defer cleanupS1()
 
@@ -215,16 +214,18 @@ func TestHeartbeat_ClearAllHeartbeatTimers(t *testing.T) {
 func TestHeartbeat_Server_HeartbeatTTL_Failover(t *testing.T) {
 	t.Parallel()
 
-	s1, cleanupS1 := TestServer(t, nil)
+	s1, cleanupS1 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 3
+	})
 	defer cleanupS1()
 
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 3
 	})
 	defer cleanupS2()
 
 	s3, cleanupS3 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 3
 	})
 	defer cleanupS3()
 	servers := []*Server{s1, s2, s3}
diff --git a/nomad/leader_test.go b/nomad/leader_test.go
index 5293b7d6359..6bec3766782 100644
--- a/nomad/leader_test.go
+++ b/nomad/leader_test.go
@@ -23,16 +23,18 @@ import (
 )
 
 func TestLeader_LeftServer(t *testing.T) {
-	s1, cleanupS1 := TestServer(t, nil)
+	s1, cleanupS1 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 3
+	})
 	defer cleanupS1()
 
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 3
 	})
 	defer cleanupS2()
 
 	s3, cleanupS3 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 3
 	})
 	defer cleanupS3()
 	servers := []*Server{s1, s2, s3}
@@ -83,16 +85,18 @@ func TestLeader_LeftServer(t *testing.T) {
 }
 
 func TestLeader_LeftLeader(t *testing.T) {
-	s1, cleanupS1 := TestServer(t, nil)
+	s1, cleanupS1 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 3
+	})
 	defer cleanupS1()
 
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 3
 	})
 	defer cleanupS2()
 
 	s3, cleanupS3 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 3
 	})
 	defer cleanupS3()
 	servers := []*Server{s1, s2, s3}
@@ -154,24 +158,29 @@ func TestLeader_MultiBootstrap(t *testing.T) {
 
 	// Ensure we don't have multiple raft peers
 	for _, s := range servers {
-		peers, _ := s.numPeers()
+		peers, err := s.numPeers()
+		if err != nil {
+			t.Fatalf("failed: %v", err)
+		}
 		if peers != 1 {
-			t.Fatalf("should only have 1 raft peer!")
+			t.Fatalf("should only have 1 raft peer! %v", peers)
 		}
 	}
 }
 
 func TestLeader_PlanQueue_Reset(t *testing.T) {
-	s1, cleanupS1 := TestServer(t, nil)
+	s1, cleanupS1 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 3
+	})
 	defer cleanupS1()
 
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 3
 	})
 	defer cleanupS2()
 
 	s3, cleanupS3 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 3
 	})
 	defer cleanupS3()
 	servers := []*Server{s1, s2, s3}
@@ -223,13 +232,13 @@ func TestLeader_EvalBroker_Reset(t *testing.T) {
 
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
 		c.NumSchedulers = 0
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 3
 	})
 	defer cleanupS2()
 
 	s3, cleanupS3 := TestServer(t, func(c *Config) {
 		c.NumSchedulers = 0
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 3
 	})
 	defer cleanupS3()
 	servers := []*Server{s1, s2, s3}
@@ -281,13 +290,13 @@ func TestLeader_PeriodicDispatcher_Restore_Adds(t *testing.T) {
 
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
 		c.NumSchedulers = 0
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 3
 	})
 	defer cleanupS2()
 
 	s3, cleanupS3 := TestServer(t, func(c *Config) {
 		c.NumSchedulers = 0
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 3
 	})
 	defer cleanupS3()
 	servers := []*Server{s1, s2, s3}
@@ -691,29 +700,27 @@ func TestLeader_ClusterID_upgradePath(t *testing.T) {
 		cleanup func()
 	}
 
-	outdated := func(bootstrap bool) server {
+	outdated := func() server {
 		s, cleanup := TestServer(t, func(c *Config) {
 			c.NumSchedulers = 0
 			c.Build = before
-			c.DevDisableBootstrap = bootstrap
 			c.BootstrapExpect = 3
 			c.Logger.SetLevel(hclog.Trace)
 		})
 		return server{s: s, cleanup: cleanup}
 	}
 
-	upgraded := func(bootstrap bool) server {
+	upgraded := func() server {
 		s, cleanup := TestServer(t, func(c *Config) {
 			c.NumSchedulers = 0
 			c.Build = after
-			c.DevDisableBootstrap = bootstrap
-			c.BootstrapExpect = 3
+			c.BootstrapExpect = 0
 			c.Logger.SetLevel(hclog.Trace)
 		})
 		return server{s: s, cleanup: cleanup}
 	}
 
-	servers := []server{outdated(false), outdated(true), outdated(true)}
+	servers := []server{outdated(), outdated(), outdated()}
 	// fallback shutdown attempt in case testing fails
 	defer servers[0].cleanup()
 	defer servers[1].cleanup()
@@ -722,7 +729,7 @@ func TestLeader_ClusterID_upgradePath(t *testing.T) {
 	upgrade := func(i int) {
 		previous := servers[i]
 
-		servers[i] = upgraded(true)
+		servers[i] = upgraded()
 		TestJoin(t, servers[i].s, servers[(i+1)%3].s, servers[(i+2)%3].s)
 		testutil.WaitForLeader(t, servers[i].s.RPC)
 
@@ -809,7 +816,6 @@ func TestLeader_ClusterID_noUpgrade(t *testing.T) {
 		c.Logger.SetLevel(hclog.Trace)
 		c.NumSchedulers = 0
 		c.Build = minClusterIDVersion.String()
-		c.DevDisableBootstrap = true
 		c.BootstrapExpect = 3
 	})
 	defer cleanupS2()
@@ -817,7 +823,6 @@ func TestLeader_ClusterID_noUpgrade(t *testing.T) {
 		c.Logger.SetLevel(hclog.Trace)
 		c.NumSchedulers = 0
 		c.Build = minClusterIDVersion.String()
-		c.DevDisableBootstrap = true
 		c.BootstrapExpect = 3
 	})
 	defer cleanupS3()
@@ -1012,13 +1017,13 @@ func TestLeader_UpgradeRaftVersion(t *testing.T) {
 	defer cleanupS1()
 
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 3
 		c.RaftConfig.ProtocolVersion = 1
 	})
 	defer cleanupS2()
 
 	s3, cleanupS3 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 3
 		c.RaftConfig.ProtocolVersion = 2
 	})
 	defer cleanupS3()
@@ -1054,7 +1059,7 @@ func TestLeader_UpgradeRaftVersion(t *testing.T) {
 
 	// Replace the dead server with one running raft protocol v3
 	s4, cleanupS4 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 3
 		c.Datacenter = "dc1"
 		c.RaftConfig.ProtocolVersion = 3
 	})
@@ -1111,14 +1116,12 @@ func leaderElectionTest(t *testing.T, raftProtocol raft.ProtocolVersion) {
 
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
 		c.BootstrapExpect = 3
-		c.DevDisableBootstrap = true
 		c.RaftConfig.ProtocolVersion = raftProtocol
 	})
 	defer cleanupS2()
 
 	s3, cleanupS3 := TestServer(t, func(c *Config) {
 		c.BootstrapExpect = 3
-		c.DevDisableBootstrap = true
 		c.RaftConfig.ProtocolVersion = raftProtocol
 	})
 	defer cleanupS3() // todo(shoenig) added this, should be here right??
@@ -1170,13 +1173,13 @@ func TestLeader_RollRaftServer(t *testing.T) {
 	defer cleanupS1()
 
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 3
 		c.RaftConfig.ProtocolVersion = 2
 	})
 	defer cleanupS2()
 
 	s3, cleanupS3 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 3
 		c.RaftConfig.ProtocolVersion = 2
 	})
 	defer cleanupS3()
@@ -1207,7 +1210,7 @@ func TestLeader_RollRaftServer(t *testing.T) {
 
 	// Replace the dead server with one running raft protocol v3
 	s4, cleanupS4 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 3
 		c.RaftConfig.ProtocolVersion = 3
 	})
 	defer cleanupS4()
@@ -1230,7 +1233,7 @@ func TestLeader_RollRaftServer(t *testing.T) {
 	}
 	// Replace another dead server with one running raft protocol v3
 	s5, cleanupS5 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 3
 		c.RaftConfig.ProtocolVersion = 3
 	})
 	defer cleanupS5()
@@ -1254,7 +1257,7 @@ func TestLeader_RollRaftServer(t *testing.T) {
 
 	// Replace the last dead server with one running raft protocol v3
 	s6, cleanupS6 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 3
 		c.RaftConfig.ProtocolVersion = 3
 	})
 	defer cleanupS6()
@@ -1330,19 +1333,22 @@ func TestServer_ReconcileMember(t *testing.T) {
 
 	// Create a three node cluster
 	s1, cleanupS1 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		// disable bootstrapping
+		c.BootstrapExpect = 0
 		c.RaftConfig.ProtocolVersion = 3
 	})
 	defer cleanupS1()
 
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		// disable bootstrapping
+		c.BootstrapExpect = 0
 		c.RaftConfig.ProtocolVersion = 3
 	})
 	defer cleanupS2()
 
 	s3, cleanupS3 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		// disable bootstrapping
+		c.BootstrapExpect = 0
 		c.RaftConfig.ProtocolVersion = 2
 	})
 	defer cleanupS3()
diff --git a/nomad/node_endpoint_test.go b/nomad/node_endpoint_test.go
index a3fb9d5b60d..9d7ebe5ecf8 100644
--- a/nomad/node_endpoint_test.go
+++ b/nomad/node_endpoint_test.go
@@ -97,7 +97,7 @@ func TestClientEndpoint_Register_NodeConn_Forwarded(t *testing.T) {
 
 	defer cleanupS1()
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 2
 	})
 	defer cleanupS2()
 	TestJoin(t, s1, s2)
@@ -754,16 +754,18 @@ func TestClientEndpoint_UpdateStatus_GetEvals(t *testing.T) {
 func TestClientEndpoint_UpdateStatus_HeartbeatOnly(t *testing.T) {
 	t.Parallel()
 
-	s1, cleanupS1 := TestServer(t, nil)
+	s1, cleanupS1 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 3
+	})
 	defer cleanupS1()
 
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 3
 	})
 	defer cleanupS2()
 
 	s3, cleanupS3 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 3
 	})
 	defer cleanupS3()
 	servers := []*Server{s1, s2, s3}
diff --git a/nomad/rpc_test.go b/nomad/rpc_test.go
index 4d900c31656..73e47a4f7a5 100644
--- a/nomad/rpc_test.go
+++ b/nomad/rpc_test.go
@@ -45,10 +45,12 @@ func rpcClient(t *testing.T, s *Server) rpc.ClientCodec {
 func TestRPC_forwardLeader(t *testing.T) {
 	t.Parallel()
 
-	s1, cleanupS1 := TestServer(t, nil)
+	s1, cleanupS1 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 2
+	})
 	defer cleanupS1()
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 2
 	})
 	defer cleanupS2()
 	TestJoin(t, s1, s2)
@@ -259,10 +261,12 @@ func TestRPC_streamingRpcConn_badMethod(t *testing.T) {
 	t.Parallel()
 	require := require.New(t)
 
-	s1, cleanupS1 := TestServer(t, nil)
+	s1, cleanupS1 := TestServer(t, func(c *Config) {
+		c.BootstrapExpect = 2
+	})
 	defer cleanupS1()
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
-		c.DevDisableBootstrap = true
+		c.BootstrapExpect = 2
 	})
 	defer cleanupS2()
 	TestJoin(t, s1, s2)
@@ -298,7 +302,6 @@ func TestRPC_streamingRpcConn_badMethod_TLS(t *testing.T) {
 		c.Region = "regionFoo"
 		c.BootstrapExpect = 2
 		c.DevMode = false
-		c.DevDisableBootstrap = true
 		c.DataDir = path.Join(dir, "node1")
 		c.TLSConfig = &config.TLSConfig{
 			EnableHTTP:           true,
@@ -315,7 +318,6 @@ func TestRPC_streamingRpcConn_badMethod_TLS(t *testing.T) {
 		c.Region = "regionFoo"
 		c.BootstrapExpect = 2
 		c.DevMode = false
-		c.DevDisableBootstrap = true
 		c.DataDir = path.Join(dir, "node2")
 		c.TLSConfig = &config.TLSConfig{
 			EnableHTTP:           true,
@@ -354,7 +356,6 @@ func TestRPC_streamingRpcConn_goodMethod_Plaintext(t *testing.T) {
 		c.Region = "regionFoo"
 		c.BootstrapExpect = 2
 		c.DevMode = false
-		c.DevDisableBootstrap = true
 		c.DataDir = path.Join(dir, "node1")
 	})
 	defer cleanupS1()
@@ -363,7 +364,6 @@ func TestRPC_streamingRpcConn_goodMethod_Plaintext(t *testing.T) {
 		c.Region = "regionFoo"
 		c.BootstrapExpect = 2
 		c.DevMode = false
-		c.DevDisableBootstrap = true
 		c.DataDir = path.Join(dir, "node2")
 	})
 	defer cleanupS2()
@@ -414,7 +414,6 @@ func TestRPC_streamingRpcConn_goodMethod_TLS(t *testing.T) {
 		c.Region = "regionFoo"
 		c.BootstrapExpect = 2
 		c.DevMode = false
-		c.DevDisableBootstrap = true
 		c.DataDir = path.Join(dir, "node1")
 		c.TLSConfig = &config.TLSConfig{
 			EnableHTTP:           true,
@@ -431,7 +430,6 @@ func TestRPC_streamingRpcConn_goodMethod_TLS(t *testing.T) {
 		c.Region = "regionFoo"
 		c.BootstrapExpect = 2
 		c.DevMode = false
-		c.DevDisableBootstrap = true
 		c.DataDir = path.Join(dir, "node2")
 		c.TLSConfig = &config.TLSConfig{
 			EnableHTTP:           true,
diff --git a/nomad/serf.go b/nomad/serf.go
index 7c00745e3cb..a955e7b1e77 100644
--- a/nomad/serf.go
+++ b/nomad/serf.go
@@ -83,7 +83,7 @@ func (s *Server) nodeJoin(me serf.MemberEvent) {
 		s.peerLock.Unlock()
 
 		// If we still expecting to bootstrap, may need to handle this
-		if atomic.LoadInt32(&s.config.BootstrapExpect) != 0 {
+		if atomic.LoadInt32(&s.config.Bootstrapped) == 0 {
 			s.maybeBootstrap()
 		}
 	}
@@ -111,7 +111,7 @@ func (s *Server) maybeBootstrap() {
 	// Bootstrap can only be done if there are no committed logs,
 	// remove our expectations of bootstrapping
 	if index != 0 {
-		atomic.StoreInt32(&s.config.BootstrapExpect, 0)
+		atomic.StoreInt32(&s.config.Bootstrapped, 1)
 		return
 	}
 
@@ -127,7 +127,7 @@ func (s *Server) maybeBootstrap() {
 		if p.Region != s.config.Region {
 			continue
 		}
-		if p.Expect != 0 && p.Expect != int(atomic.LoadInt32(&s.config.BootstrapExpect)) {
+		if p.Expect != 0 && p.Expect != s.config.BootstrapExpect {
 			s.logger.Error("peer has a conflicting expect value. All nodes should expect the same number", "member", member)
 			return
 		}
@@ -138,11 +138,12 @@ func (s *Server) maybeBootstrap() {
 		if !p.NonVoter {
 			voters++
 		}
+
 		servers = append(servers, *p)
 	}
 
 	// Skip if we haven't met the minimum expect count
-	if voters < int(atomic.LoadInt32(&s.config.BootstrapExpect)) {
+	if voters < s.config.BootstrapExpect {
 		return
 	}
 
@@ -181,7 +182,7 @@ func (s *Server) maybeBootstrap() {
 		if len(peers) > 0 {
 			s.logger.Info("disabling bootstrap mode because existing Raft peers being reported by peer",
 				"peer_name", server.Name, "peer_address", server.Addr)
-			atomic.StoreInt32(&s.config.BootstrapExpect, 0)
+			atomic.StoreInt32(&s.config.Bootstrapped, 1)
 			return
 		}
 	}
@@ -223,7 +224,7 @@ func (s *Server) maybeBootstrap() {
 	}
 
 	// Bootstrapping complete, or failed for some reason, don't enter this again
-	atomic.StoreInt32(&s.config.BootstrapExpect, 0)
+	atomic.StoreInt32(&s.config.Bootstrapped, 1)
 }
 
 // nodeFailed is used to handle fail events on the serf cluster
diff --git a/nomad/serf_test.go b/nomad/serf_test.go
index d66dbc38248..56181e9b707 100644
--- a/nomad/serf_test.go
+++ b/nomad/serf_test.go
@@ -105,7 +105,6 @@ func TestNomad_ReapPeer(t *testing.T) {
 		c.NodeName = "node1"
 		c.BootstrapExpect = 3
 		c.DevMode = false
-		c.DevDisableBootstrap = true
 		c.DataDir = path.Join(dir, "node1")
 	})
 	defer cleanupS1()
@@ -113,7 +112,6 @@ func TestNomad_ReapPeer(t *testing.T) {
 		c.NodeName = "node2"
 		c.BootstrapExpect = 3
 		c.DevMode = false
-		c.DevDisableBootstrap = true
 		c.DataDir = path.Join(dir, "node2")
 	})
 	defer cleanupS2()
@@ -121,7 +119,6 @@ func TestNomad_ReapPeer(t *testing.T) {
 		c.NodeName = "node3"
 		c.BootstrapExpect = 3
 		c.DevMode = false
-		c.DevDisableBootstrap = true
 		c.DataDir = path.Join(dir, "node3")
 	})
 	defer cleanupS3()
@@ -200,21 +197,18 @@ func TestNomad_BootstrapExpect(t *testing.T) {
 	s1, cleanupS1 := TestServer(t, func(c *Config) {
 		c.BootstrapExpect = 3
 		c.DevMode = false
-		c.DevDisableBootstrap = true
 		c.DataDir = path.Join(dir, "node1")
 	})
 	defer cleanupS1()
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
 		c.BootstrapExpect = 3
 		c.DevMode = false
-		c.DevDisableBootstrap = true
 		c.DataDir = path.Join(dir, "node2")
 	})
 	defer cleanupS2()
 	s3, cleanupS3 := TestServer(t, func(c *Config) {
 		c.BootstrapExpect = 3
 		c.DevMode = false
-		c.DevDisableBootstrap = true
 		c.DataDir = path.Join(dir, "node3")
 	})
 	defer cleanupS3()
@@ -263,7 +257,6 @@ func TestNomad_BootstrapExpect(t *testing.T) {
 	s4, cleanupS4 := TestServer(t, func(c *Config) {
 		c.BootstrapExpect = 3
 		c.DevMode = false
-		c.DevDisableBootstrap = true
 		c.DataDir = path.Join(dir, "node4")
 	})
 	defer cleanupS4()
@@ -313,7 +306,6 @@ func TestNomad_BootstrapExpect_NonVoter(t *testing.T) {
 	s1, cleanupS1 := TestServer(t, func(c *Config) {
 		c.BootstrapExpect = 2
 		c.DevMode = false
-		c.DevDisableBootstrap = true
 		c.DataDir = path.Join(dir, "node1")
 		c.NonVoter = true
 	})
@@ -321,7 +313,6 @@ func TestNomad_BootstrapExpect_NonVoter(t *testing.T) {
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
 		c.BootstrapExpect = 2
 		c.DevMode = false
-		c.DevDisableBootstrap = true
 		c.DataDir = path.Join(dir, "node2")
 		c.NonVoter = true
 	})
@@ -329,7 +320,6 @@ func TestNomad_BootstrapExpect_NonVoter(t *testing.T) {
 	s3, cleanupS3 := TestServer(t, func(c *Config) {
 		c.BootstrapExpect = 2
 		c.DevMode = false
-		c.DevDisableBootstrap = true
 		c.DataDir = path.Join(dir, "node3")
 	})
 	defer cleanupS3()
@@ -351,7 +341,6 @@ func TestNomad_BootstrapExpect_NonVoter(t *testing.T) {
 	s4, cleanupS4 := TestServer(t, func(c *Config) {
 		c.BootstrapExpect = 2
 		c.DevMode = false
-		c.DevDisableBootstrap = true
 		c.DataDir = path.Join(dir, "node4")
 	})
 	defer cleanupS4()
@@ -418,12 +407,10 @@ func TestNomad_BadExpect(t *testing.T) {
 
 	s1, cleanupS1 := TestServer(t, func(c *Config) {
 		c.BootstrapExpect = 2
-		c.DevDisableBootstrap = true
 	})
 	defer cleanupS1()
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
 		c.BootstrapExpect = 3
-		c.DevDisableBootstrap = true
 	})
 	defer cleanupS2()
 	servers := []*Server{s1, s2}
@@ -446,7 +433,7 @@ func TestNomad_BadExpect(t *testing.T) {
 	testutil.WaitForResult(func() (bool, error) {
 		for _, s := range servers {
 			p, _ := s.numPeers()
-			if p != 1 {
+			if p != 0 {
 				return false, fmt.Errorf("%d", p)
 			}
 		}
diff --git a/nomad/server.go b/nomad/server.go
index 3ddb4a586ad..0fb95e05d4a 100644
--- a/nomad/server.go
+++ b/nomad/server.go
@@ -822,7 +822,7 @@ func (s *Server) setupBootstrapHandler() error {
 
 		// (ab)use serf.go's behavior of setting BootstrapExpect to
 		// zero if we have bootstrapped.  If we have bootstrapped
-		bootstrapExpect := atomic.LoadInt32(&s.config.BootstrapExpect)
+		bootstrapExpect := s.config.BootstrapExpect
 		if bootstrapExpect == 0 {
 			// This Nomad Server has been bootstrapped.  Rely on
 			// the peersTimeout firing as a guard to prevent
@@ -848,7 +848,7 @@ func (s *Server) setupBootstrapHandler() error {
 			// quorum has been reached, we do not need to poll
 			// Consul.  Let the normal timeout-based strategy
 			// take over.
-			if raftPeers >= int(bootstrapExpect) {
+			if raftPeers >= bootstrapExpect {
 				peersTimeout.Reset(peersPollInterval + lib.RandomStagger(peersPollInterval/peersPollJitterFactor))
 				return nil
 			}
@@ -1275,9 +1275,9 @@ func (s *Server) setupRaft() error {
 		}
 	}
 
-	// If we are in bootstrap or dev mode and the state is clean then we can
+	// If we are a single server cluster and the state is clean then we can
 	// bootstrap now.
-	if s.config.Bootstrap || s.config.DevMode {
+	if s.isSingleServerCluster() {
 		hasState, err := raft.HasExistingState(log, stable, snap)
 		if err != nil {
 			return err
@@ -1320,10 +1320,10 @@ func (s *Server) setupSerf(conf *serf.Config, ch chan serf.Event, path string) (
 	conf.Tags["id"] = s.config.NodeID
 	conf.Tags["rpc_addr"] = s.clientRpcAdvertise.(*net.TCPAddr).IP.String()         // Address that clients will use to RPC to servers
 	conf.Tags["port"] = fmt.Sprintf("%d", s.serverRpcAdvertise.(*net.TCPAddr).Port) // Port servers use to RPC to one and another
-	if s.config.Bootstrap || (s.config.DevMode && !s.config.DevDisableBootstrap) {
+	if s.isSingleServerCluster() {
 		conf.Tags["bootstrap"] = "1"
 	}
-	bootstrapExpect := atomic.LoadInt32(&s.config.BootstrapExpect)
+	bootstrapExpect := s.config.BootstrapExpect
 	if bootstrapExpect != 0 {
 		conf.Tags["expect"] = fmt.Sprintf("%d", bootstrapExpect)
 	}
@@ -1524,7 +1524,7 @@ func (s *Server) Stats() map[string]map[string]string {
 			"server":        "true",
 			"leader":        fmt.Sprintf("%v", s.IsLeader()),
 			"leader_addr":   string(s.raft.Leader()),
-			"bootstrap":     fmt.Sprintf("%v", s.config.Bootstrap),
+			"bootstrap":     fmt.Sprintf("%v", s.isSingleServerCluster()),
 			"known_regions": toString(uint64(len(s.peers))),
 		},
 		"raft":    s.raft.Stats(),
@@ -1617,6 +1617,10 @@ func (s *Server) ClusterID() (string, error) {
 	return generatedID, nil
 }
 
+func (s *Server) isSingleServerCluster() bool {
+	return s.config.BootstrapExpect == 1
+}
+
 // peersInfoContent is used to help operators understand what happened to the
 // peers.json file. This is written to a file called peers.info in the same
 // location.
diff --git a/nomad/server_test.go b/nomad/server_test.go
index 797b57e62eb..64e34de576b 100644
--- a/nomad/server_test.go
+++ b/nomad/server_test.go
@@ -56,7 +56,6 @@ func TestServer_RPC_TLS(t *testing.T) {
 		c.Region = "regionFoo"
 		c.BootstrapExpect = 3
 		c.DevMode = false
-		c.DevDisableBootstrap = true
 		c.DataDir = path.Join(dir, "node1")
 		c.TLSConfig = &config.TLSConfig{
 			EnableHTTP:           true,
@@ -73,7 +72,6 @@ func TestServer_RPC_TLS(t *testing.T) {
 		c.Region = "regionFoo"
 		c.BootstrapExpect = 3
 		c.DevMode = false
-		c.DevDisableBootstrap = true
 		c.DataDir = path.Join(dir, "node2")
 		c.TLSConfig = &config.TLSConfig{
 			EnableHTTP:           true,
@@ -90,7 +88,6 @@ func TestServer_RPC_TLS(t *testing.T) {
 		c.Region = "regionFoo"
 		c.BootstrapExpect = 3
 		c.DevMode = false
-		c.DevDisableBootstrap = true
 		c.DataDir = path.Join(dir, "node3")
 		c.TLSConfig = &config.TLSConfig{
 			EnableHTTP:           true,
@@ -125,7 +122,6 @@ func TestServer_RPC_MixedTLS(t *testing.T) {
 		c.Region = "regionFoo"
 		c.BootstrapExpect = 3
 		c.DevMode = false
-		c.DevDisableBootstrap = true
 		c.DataDir = path.Join(dir, "node1")
 		c.TLSConfig = &config.TLSConfig{
 			EnableHTTP:           true,
@@ -142,7 +138,6 @@ func TestServer_RPC_MixedTLS(t *testing.T) {
 		c.Region = "regionFoo"
 		c.BootstrapExpect = 3
 		c.DevMode = false
-		c.DevDisableBootstrap = true
 		c.DataDir = path.Join(dir, "node2")
 		c.TLSConfig = &config.TLSConfig{
 			EnableHTTP:           true,
@@ -159,7 +154,6 @@ func TestServer_RPC_MixedTLS(t *testing.T) {
 		c.Region = "regionFoo"
 		c.BootstrapExpect = 3
 		c.DevMode = false
-		c.DevDisableBootstrap = true
 		c.DataDir = path.Join(dir, "node3")
 	})
 	defer cleanupS3()
@@ -470,7 +464,6 @@ func TestServer_Reload_TLSConnections_Raft(t *testing.T) {
 	s1, cleanupS1 := TestServer(t, func(c *Config) {
 		c.BootstrapExpect = 2
 		c.DevMode = false
-		c.DevDisableBootstrap = true
 		c.DataDir = path.Join(dir, "node1")
 		c.NodeName = "node1"
 		c.Region = "regionFoo"
@@ -480,7 +473,6 @@ func TestServer_Reload_TLSConnections_Raft(t *testing.T) {
 	s2, cleanupS2 := TestServer(t, func(c *Config) {
 		c.BootstrapExpect = 2
 		c.DevMode = false
-		c.DevDisableBootstrap = true
 		c.DataDir = path.Join(dir, "node2")
 		c.NodeName = "node2"
 		c.Region = "regionFoo"
diff --git a/nomad/stats_fetcher_test.go b/nomad/stats_fetcher_test.go
index 0278af94299..3c508c73ad3 100644
--- a/nomad/stats_fetcher_test.go
+++ b/nomad/stats_fetcher_test.go
@@ -13,7 +13,6 @@ func TestStatsFetcher(t *testing.T) {
 
 	conf := func(c *Config) {
 		c.Region = "region-a"
-		c.DevDisableBootstrap = true
 		c.BootstrapExpect = 3
 	}
 
diff --git a/nomad/testing.go b/nomad/testing.go
index 8110fb954c1..3beeeb3702c 100644
--- a/nomad/testing.go
+++ b/nomad/testing.go
@@ -45,6 +45,7 @@ func TestServer(t testing.T, cb func(*Config)) (*Server, func()) {
 	config.Logger = testlog.HCLogger(t)
 	config.Build = version.Version + "+unittest"
 	config.DevMode = true
+	config.BootstrapExpect = 1
 	nodeNum := atomic.AddUint32(&nodeNumber, 1)
 	config.NodeName = fmt.Sprintf("nomad-%03d", nodeNum)
 

From d5c7d6e491e36a11159211f5236c19a41bed4d8e Mon Sep 17 00:00:00 2001
From: Mahmood Ali <mahmood@hashicorp.com>
Date: Mon, 2 Mar 2020 11:35:28 -0500
Subject: [PATCH 2/3] avoid forwarding leadership checks in tests

The tests only care if a test server recognizes the leader.
---
 testutil/wait.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/testutil/wait.go b/testutil/wait.go
index 45e23e6b5a9..6ab3ea47211 100644
--- a/testutil/wait.go
+++ b/testutil/wait.go
@@ -85,6 +85,7 @@ type rpcFn func(string, interface{}, interface{}) error
 func WaitForLeader(t testing.T, rpc rpcFn) {
 	WaitForResult(func() (bool, error) {
 		args := &structs.GenericRequest{}
+		args.AllowStale = true
 		var leader string
 		err := rpc("Status.Leader", args, &leader)
 		return leader != "", err

From 5e635bb27671948e84cd06acc495526499705b10 Mon Sep 17 00:00:00 2001
From: Mahmood Ali <mahmood@hashicorp.com>
Date: Tue, 3 Mar 2020 16:55:54 -0500
Subject: [PATCH 3/3] fix typo

---
 nomad/config.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nomad/config.go b/nomad/config.go
index c15268afd98..8a0fb7a181b 100644
--- a/nomad/config.go
+++ b/nomad/config.go
@@ -49,7 +49,7 @@ func DefaultRPCAddr() *net.TCPAddr {
 
 // Config is used to parameterize the server
 type Config struct {
-	// Bootstrapped indictes if Server has bootstrapped or not.
+	// Bootstrapped indicates if Server has bootstrapped or not.
 	// Its value must be 0 (not bootstrapped) or 1 (bootstrapped).
 	// All operations on Bootstrapped must be handled via `atomic.*Int32()` calls
 	Bootstrapped int32