diff --git a/client/rpc_test.go b/client/rpc_test.go index 943ca6f1e61..2f8c1337874 100644 --- a/client/rpc_test.go +++ b/client/rpc_test.go @@ -63,7 +63,6 @@ func TestRpc_streamingRpcConn_badEndpoint_TLS(t *testing.T) { s1, cleanupS1 := nomad.TestServer(t, func(c *nomad.Config) { c.Region = "regionFoo" c.BootstrapExpect = 1 - c.DevDisableBootstrap = true c.TLSConfig = &sconfig.TLSConfig{ EnableHTTP: true, EnableRPC: true, diff --git a/command/agent/agent.go b/command/agent/agent.go index c918b03e797..04eff1cc667 100644 --- a/command/agent/agent.go +++ b/command/agent/agent.go @@ -11,7 +11,6 @@ import ( "runtime" "strings" "sync" - "sync/atomic" "time" metrics "github.com/armon/go-metrics" @@ -157,11 +156,7 @@ func convertServerConfig(agentConfig *Config) (*nomad.Config, error) { conf.NodeName = agentConfig.NodeName } if agentConfig.Server.BootstrapExpect > 0 { - if agentConfig.Server.BootstrapExpect == 1 { - conf.Bootstrap = true - } else { - atomic.StoreInt32(&conf.BootstrapExpect, int32(agentConfig.Server.BootstrapExpect)) - } + conf.BootstrapExpect = agentConfig.Server.BootstrapExpect } if agentConfig.DataDir != "" { conf.DataDir = filepath.Join(agentConfig.DataDir, "server") diff --git a/command/agent/agent_test.go b/command/agent/agent_test.go index 09a5cde4e3d..9dcd02a98b8 100644 --- a/command/agent/agent_test.go +++ b/command/agent/agent_test.go @@ -166,14 +166,12 @@ func TestAgent_ServerConfig(t *testing.T) { conf.Server.BootstrapExpect = 1 out, err = a.serverConfig() require.NoError(t, err) - require.True(t, out.Bootstrap) - require.Equal(t, int32(0), out.BootstrapExpect) + require.Equal(t, 1, out.BootstrapExpect) conf.Server.BootstrapExpect = 3 out, err = a.serverConfig() require.NoError(t, err) - require.False(t, out.Bootstrap) - require.Equal(t, int32(3), out.BootstrapExpect) + require.Equal(t, 3, out.BootstrapExpect) } func TestAgent_ServerConfig_SchedulerFlags(t *testing.T) { diff --git a/command/agent/config.go b/command/agent/config.go index 2e560ac423d..2df63294c16 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -778,7 +778,8 @@ func DevConfig(mode *devModeConfig) *Config { conf.LogLevel = "DEBUG" conf.Client.Enabled = true conf.Server.Enabled = true - conf.DevMode = mode != nil + conf.DevMode = true + conf.Server.BootstrapExpect = 1 conf.EnableDebug = true conf.DisableAnonymousSignature = true conf.Consul.AutoAdvertise = helper.BoolToPtr(true) diff --git a/command/agent/testagent.go b/command/agent/testagent.go index b2e317574c6..64fb3193db5 100644 --- a/command/agent/testagent.go +++ b/command/agent/testagent.go @@ -168,7 +168,7 @@ RETRY: } failed := false - if a.Config.NomadConfig.Bootstrap && a.Config.Server.Enabled { + if a.Config.NomadConfig.BootstrapExpect == 1 && a.Config.Server.Enabled { testutil.WaitForResult(func() (bool, error) { args := &structs.GenericRequest{} var leader string @@ -358,10 +358,6 @@ func (a *TestAgent) config() *Config { config.ServerHealthInterval = 50 * time.Millisecond config.AutopilotInterval = 100 * time.Millisecond - // Bootstrap ourselves - config.Bootstrap = true - config.BootstrapExpect = 1 - // Tighten the fingerprinter timeouts if conf.Client.Options == nil { conf.Client.Options = make(map[string]string) diff --git a/nomad/acl_endpoint_test.go b/nomad/acl_endpoint_test.go index 1d83d13b464..7cf5c63998f 100644 --- a/nomad/acl_endpoint_test.go +++ b/nomad/acl_endpoint_test.go @@ -1077,8 +1077,6 @@ func TestACLEndpoint_Bootstrap_Reset(t *testing.T) { c.ACLEnabled = true c.DataDir = dir c.DevMode = false - c.Bootstrap = true - c.DevDisableBootstrap = false }) defer cleanupS1() codec := rpcClient(t, s1) diff --git a/nomad/autopilot_test.go b/nomad/autopilot_test.go index c46a9cb5159..bd2715b4690 100644 --- a/nomad/autopilot_test.go +++ b/nomad/autopilot_test.go @@ -73,7 +73,6 @@ func TestAutopilot_CleanupDeadServer(t *testing.T) { func testCleanupDeadServer(t *testing.T, raftVersion int) { conf := func(c *Config) { - c.DevDisableBootstrap = true c.BootstrapExpect = 3 c.RaftConfig.ProtocolVersion = raft.ProtocolVersion(raftVersion) } @@ -127,13 +126,13 @@ func testCleanupDeadServer(t *testing.T, raftVersion int) { func TestAutopilot_CleanupDeadServerPeriodic(t *testing.T) { t.Parallel() - s1, cleanupS1 := TestServer(t, nil) - defer cleanupS1() - conf := func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 5 } + s1, cleanupS1 := TestServer(t, conf) + defer cleanupS1() + s2, cleanupS2 := TestServer(t, conf) defer cleanupS2() @@ -174,16 +173,14 @@ func TestAutopilot_CleanupDeadServerPeriodic(t *testing.T) { func TestAutopilot_RollingUpdate(t *testing.T) { t.Parallel() - s1, cleanupS1 := TestServer(t, func(c *Config) { - c.RaftConfig.ProtocolVersion = 3 - }) - defer cleanupS1() - conf := func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 3 c.RaftConfig.ProtocolVersion = 3 } + s1, cleanupS1 := TestServer(t, conf) + defer cleanupS1() + s2, cleanupS2 := TestServer(t, conf) defer cleanupS2() @@ -248,19 +245,21 @@ func TestAutopilot_CleanupStaleRaftServer(t *testing.T) { t.Skip("TestAutopilot_CleanupDeadServer is very flaky, removing it for now") t.Parallel() - s1, cleanupS1 := TestServer(t, nil) - defer cleanupS1() - conf := func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 3 } + s1, cleanupS1 := TestServer(t, conf) + defer cleanupS1() + s2, cleanupS2 := TestServer(t, conf) defer cleanupS2() s3, cleanupS3 := TestServer(t, conf) defer cleanupS3() - s4, cleanupS4 := TestServer(t, conf) + s4, cleanupS4 := TestServer(t, func(c *Config) { + c.BootstrapExpect = 0 + }) defer cleanupS4() servers := []*Server{s1, s2, s3} @@ -304,7 +303,7 @@ func TestAutopilot_PromoteNonVoter(t *testing.T) { testutil.WaitForLeader(t, s1.RPC) s2, cleanupS2 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 0 c.RaftConfig.ProtocolVersion = 3 }) defer cleanupS2() diff --git a/nomad/client_agent_endpoint_test.go b/nomad/client_agent_endpoint_test.go index c87cf51b2d9..bc295d00a0b 100644 --- a/nomad/client_agent_endpoint_test.go +++ b/nomad/client_agent_endpoint_test.go @@ -30,10 +30,12 @@ func TestMonitor_Monitor_Remote_Client(t *testing.T) { require := require.New(t) // start server and client - s1, cleanupS1 := TestServer(t, nil) + s1, cleanupS1 := TestServer(t, func(c *Config) { + c.BootstrapExpect = 2 + }) defer cleanupS1() s2, cleanupS2 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 2 }) defer cleanupS2() TestJoin(t, s1, s2) @@ -125,15 +127,16 @@ func TestMonitor_Monitor_RemoteServer(t *testing.T) { foreignRegion := "foo" // start servers - s1, cleanupS1 := TestServer(t, nil) + s1, cleanupS1 := TestServer(t, func(c *Config) { + c.BootstrapExpect = 2 + }) defer cleanupS1() s2, cleanupS2 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 2 }) defer cleanupS2() s3, cleanupS3 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true c.Region = foreignRegion }) defer cleanupS3() @@ -516,12 +519,12 @@ func TestAgentProfile_RemoteClient(t *testing.T) { // start server and client s1, cleanup := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 2 }) defer cleanup() s2, cleanup := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 2 }) defer cleanup() @@ -640,12 +643,13 @@ func TestAgentProfile_Server(t *testing.T) { // start servers s1, cleanup := TestServer(t, func(c *Config) { + c.BootstrapExpect = 2 c.EnableDebug = true }) defer cleanup() s2, cleanup := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 2 c.EnableDebug = true }) defer cleanup() diff --git a/nomad/client_alloc_endpoint_test.go b/nomad/client_alloc_endpoint_test.go index 08a9f148b16..49e665a23c0 100644 --- a/nomad/client_alloc_endpoint_test.go +++ b/nomad/client_alloc_endpoint_test.go @@ -186,10 +186,12 @@ func TestClientAllocations_GarbageCollectAll_Remote(t *testing.T) { require := require.New(t) // Start a server and client - s1, cleanupS1 := TestServer(t, nil) + s1, cleanupS1 := TestServer(t, func(c *Config) { + c.BootstrapExpect = 2 + }) defer cleanupS1() s2, cleanupS2 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 2 }) defer cleanupS2() TestJoin(t, s1, s2) @@ -432,10 +434,12 @@ func TestClientAllocations_GarbageCollect_Remote(t *testing.T) { require := require.New(t) // Start a server and client - s1, cleanupS1 := TestServer(t, nil) + s1, cleanupS1 := TestServer(t, func(c *Config) { + c.BootstrapExpect = 2 + }) defer cleanupS1() s2, cleanupS2 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 2 }) defer cleanupS2() TestJoin(t, s1, s2) @@ -720,10 +724,12 @@ func TestClientAllocations_Stats_Remote(t *testing.T) { require := require.New(t) // Start a server and client - s1, cleanupS1 := TestServer(t, nil) + s1, cleanupS1 := TestServer(t, func(c *Config) { + c.BootstrapExpect = 2 + }) defer cleanupS1() s2, cleanupS2 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 2 }) defer cleanupS2() TestJoin(t, s1, s2) @@ -915,10 +921,12 @@ func TestClientAllocations_Restart_Remote(t *testing.T) { require := require.New(t) // Start a server and client - s1, cleanupS1 := TestServer(t, nil) + s1, cleanupS1 := TestServer(t, func(c *Config) { + c.BootstrapExpect = 2 + }) defer cleanupS1() s2, cleanupS2 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 2 }) defer cleanupS2() TestJoin(t, s1, s2) @@ -1071,11 +1079,13 @@ func TestAlloc_ExecStreaming(t *testing.T) { t.Parallel() ////// Nomad clusters topology - not specific to test - localServer, cleanupLS := TestServer(t, nil) + localServer, cleanupLS := TestServer(t, func(c *Config) { + c.BootstrapExpect = 2 + }) defer cleanupLS() remoteServer, cleanupRS := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 2 }) defer cleanupRS() diff --git a/nomad/client_fs_endpoint_test.go b/nomad/client_fs_endpoint_test.go index 821e8abfa21..d07741db4f7 100644 --- a/nomad/client_fs_endpoint_test.go +++ b/nomad/client_fs_endpoint_test.go @@ -177,10 +177,12 @@ func TestClientFS_List_Remote(t *testing.T) { require := require.New(t) // Start a server and client - s1, cleanupS1 := TestServer(t, nil) + s1, cleanupS1 := TestServer(t, func(c *Config) { + c.BootstrapExpect = 2 + }) defer cleanupS1() s2, cleanupS2 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 2 }) defer cleanupS2() TestJoin(t, s1, s2) @@ -451,10 +453,12 @@ func TestClientFS_Stat_Remote(t *testing.T) { require := require.New(t) // Start a server and client - s1, cleanupS1 := TestServer(t, nil) + s1, cleanupS1 := TestServer(t, func(c *Config) { + c.BootstrapExpect = 2 + }) defer cleanupS1() s2, cleanupS2 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 2 }) defer cleanupS2() TestJoin(t, s1, s2) @@ -1000,10 +1004,12 @@ func TestClientFS_Streaming_Remote_Server(t *testing.T) { require := require.New(t) // Start a server and client - s1, cleanupS1 := TestServer(t, nil) + s1, cleanupS1 := TestServer(t, func(c *Config) { + c.BootstrapExpect = 2 + }) defer cleanupS1() s2, cleanupS2 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 2 }) defer cleanupS2() TestJoin(t, s1, s2) @@ -1829,10 +1835,12 @@ func TestClientFS_Logs_Remote_Server(t *testing.T) { require := require.New(t) // Start a server and client - s1, cleanupS1 := TestServer(t, nil) + s1, cleanupS1 := TestServer(t, func(c *Config) { + c.BootstrapExpect = 2 + }) defer cleanupS1() s2, cleanupS2 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 2 }) defer cleanupS2() TestJoin(t, s1, s2) diff --git a/nomad/client_rpc_test.go b/nomad/client_rpc_test.go index 6d04805793c..a653440280e 100644 --- a/nomad/client_rpc_test.go +++ b/nomad/client_rpc_test.go @@ -88,10 +88,12 @@ func TestServerWithNodeConn_NoPath(t *testing.T) { t.Parallel() require := require.New(t) - s1, cleanupS1 := TestServer(t, nil) + s1, cleanupS1 := TestServer(t, func(c *Config) { + c.BootstrapExpect = 2 + }) defer cleanupS1() s2, cleanupS2 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 2 }) defer cleanupS2() TestJoin(t, s1, s2) @@ -122,10 +124,12 @@ func TestServerWithNodeConn_Path(t *testing.T) { t.Parallel() require := require.New(t) - s1, cleanupS1 := TestServer(t, nil) + s1, cleanupS1 := TestServer(t, func(c *Config) { + c.BootstrapExpect = 2 + }) defer cleanupS1() s2, cleanupS2 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 2 }) defer cleanupS2() TestJoin(t, s1, s2) @@ -174,14 +178,16 @@ func TestServerWithNodeConn_Path_Newest(t *testing.T) { t.Parallel() require := require.New(t) - s1, cleanupS1 := TestServer(t, nil) + s1, cleanupS1 := TestServer(t, func(c *Config) { + c.BootstrapExpect = 3 + }) defer cleanupS1() s2, cleanupS2 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 3 }) defer cleanupS2() s3, cleanupS3 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 3 }) defer cleanupS3() TestJoin(t, s1, s2, s3) @@ -208,14 +214,16 @@ func TestServerWithNodeConn_PathAndErr(t *testing.T) { t.Parallel() require := require.New(t) - s1, cleanupS1 := TestServer(t, nil) + s1, cleanupS1 := TestServer(t, func(c *Config) { + c.BootstrapExpect = 3 + }) defer cleanupS1() s2, cleanupS2 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 3 }) defer cleanupS2() s3, cleanupS3 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 3 }) defer cleanupS3() TestJoin(t, s1, s2, s3) @@ -242,14 +250,16 @@ func TestServerWithNodeConn_NoPathAndErr(t *testing.T) { t.Parallel() require := require.New(t) - s1, cleanupS1 := TestServer(t, nil) + s1, cleanupS1 := TestServer(t, func(c *Config) { + c.BootstrapExpect = 3 + }) defer cleanupS1() s2, cleanupS2 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 3 }) defer cleanupS2() s3, cleanupS3 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 3 }) defer cleanupS3() TestJoin(t, s1, s2, s3) diff --git a/nomad/client_stats_endpoint_test.go b/nomad/client_stats_endpoint_test.go index 6e37ef5ba76..d27b418b93f 100644 --- a/nomad/client_stats_endpoint_test.go +++ b/nomad/client_stats_endpoint_test.go @@ -173,10 +173,12 @@ func TestClientStats_Stats_Remote(t *testing.T) { require := require.New(t) // Start a server and client - s1, cleanupS1 := TestServer(t, nil) + s1, cleanupS1 := TestServer(t, func(c *Config) { + c.BootstrapExpect = 2 + }) defer cleanupS1() s2, cleanupS2 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 2 }) defer cleanupS2() TestJoin(t, s1, s2) diff --git a/nomad/config.go b/nomad/config.go index 96d6adfa63a..8a0fb7a181b 100644 --- a/nomad/config.go +++ b/nomad/config.go @@ -49,16 +49,23 @@ func DefaultRPCAddr() *net.TCPAddr { // Config is used to parameterize the server type Config struct { - // Bootstrap mode is used to bring up the first Nomad server. It is - // required so that it can elect a leader without any other nodes - // being present - Bootstrap bool + // Bootstrapped indicates if Server has bootstrapped or not. + // Its value must be 0 (not bootstrapped) or 1 (bootstrapped). + // All operations on Bootstrapped must be handled via `atomic.*Int32()` calls + Bootstrapped int32 // BootstrapExpect mode is used to automatically bring up a // collection of Nomad servers. This can be used to automatically - // bring up a collection of nodes. All operations on BootstrapExpect - // must be handled via `atomic.*Int32()` calls. - BootstrapExpect int32 + // bring up a collection of nodes. + // + // The BootstrapExpect can be of any of the following values: + // 1: Server will form a single node cluster and become a leader immediately + // N, larger than 1: Server will wait until it's connected to N servers + // before attempting leadership and forming the cluster. No Raft Log operation + // will succeed until then. + // 0: Server will wait to get a Raft configuration from another node and may not + // attempt to form a cluster or establish leadership on its own. + BootstrapExpect int // DataDir is the directory to store our state in DataDir string @@ -71,10 +78,6 @@ type Config struct { // in the absence of ACLs EnableDebug bool - // DevDisableBootstrap is used to disable bootstrap mode while - // in DevMode. This is largely used for testing. - DevDisableBootstrap bool - // LogOutput is the location to write logs to. If this is not set, // logs will go to stderr. LogOutput io.Writer diff --git a/nomad/heartbeat_test.go b/nomad/heartbeat_test.go index f687a4ab485..02dbf79a162 100644 --- a/nomad/heartbeat_test.go +++ b/nomad/heartbeat_test.go @@ -69,7 +69,6 @@ func TestHeartbeat_ResetHeartbeatTimer_Nonleader(t *testing.T) { s1, cleanupS1 := TestServer(t, func(c *Config) { c.BootstrapExpect = 3 // Won't become leader - c.DevDisableBootstrap = true }) defer cleanupS1() @@ -215,16 +214,18 @@ func TestHeartbeat_ClearAllHeartbeatTimers(t *testing.T) { func TestHeartbeat_Server_HeartbeatTTL_Failover(t *testing.T) { t.Parallel() - s1, cleanupS1 := TestServer(t, nil) + s1, cleanupS1 := TestServer(t, func(c *Config) { + c.BootstrapExpect = 3 + }) defer cleanupS1() s2, cleanupS2 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 3 }) defer cleanupS2() s3, cleanupS3 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 3 }) defer cleanupS3() servers := []*Server{s1, s2, s3} diff --git a/nomad/leader_test.go b/nomad/leader_test.go index 5293b7d6359..6bec3766782 100644 --- a/nomad/leader_test.go +++ b/nomad/leader_test.go @@ -23,16 +23,18 @@ import ( ) func TestLeader_LeftServer(t *testing.T) { - s1, cleanupS1 := TestServer(t, nil) + s1, cleanupS1 := TestServer(t, func(c *Config) { + c.BootstrapExpect = 3 + }) defer cleanupS1() s2, cleanupS2 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 3 }) defer cleanupS2() s3, cleanupS3 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 3 }) defer cleanupS3() servers := []*Server{s1, s2, s3} @@ -83,16 +85,18 @@ func TestLeader_LeftServer(t *testing.T) { } func TestLeader_LeftLeader(t *testing.T) { - s1, cleanupS1 := TestServer(t, nil) + s1, cleanupS1 := TestServer(t, func(c *Config) { + c.BootstrapExpect = 3 + }) defer cleanupS1() s2, cleanupS2 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 3 }) defer cleanupS2() s3, cleanupS3 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 3 }) defer cleanupS3() servers := []*Server{s1, s2, s3} @@ -154,24 +158,29 @@ func TestLeader_MultiBootstrap(t *testing.T) { // Ensure we don't have multiple raft peers for _, s := range servers { - peers, _ := s.numPeers() + peers, err := s.numPeers() + if err != nil { + t.Fatalf("failed: %v", err) + } if peers != 1 { - t.Fatalf("should only have 1 raft peer!") + t.Fatalf("should only have 1 raft peer! %v", peers) } } } func TestLeader_PlanQueue_Reset(t *testing.T) { - s1, cleanupS1 := TestServer(t, nil) + s1, cleanupS1 := TestServer(t, func(c *Config) { + c.BootstrapExpect = 3 + }) defer cleanupS1() s2, cleanupS2 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 3 }) defer cleanupS2() s3, cleanupS3 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 3 }) defer cleanupS3() servers := []*Server{s1, s2, s3} @@ -223,13 +232,13 @@ func TestLeader_EvalBroker_Reset(t *testing.T) { s2, cleanupS2 := TestServer(t, func(c *Config) { c.NumSchedulers = 0 - c.DevDisableBootstrap = true + c.BootstrapExpect = 3 }) defer cleanupS2() s3, cleanupS3 := TestServer(t, func(c *Config) { c.NumSchedulers = 0 - c.DevDisableBootstrap = true + c.BootstrapExpect = 3 }) defer cleanupS3() servers := []*Server{s1, s2, s3} @@ -281,13 +290,13 @@ func TestLeader_PeriodicDispatcher_Restore_Adds(t *testing.T) { s2, cleanupS2 := TestServer(t, func(c *Config) { c.NumSchedulers = 0 - c.DevDisableBootstrap = true + c.BootstrapExpect = 3 }) defer cleanupS2() s3, cleanupS3 := TestServer(t, func(c *Config) { c.NumSchedulers = 0 - c.DevDisableBootstrap = true + c.BootstrapExpect = 3 }) defer cleanupS3() servers := []*Server{s1, s2, s3} @@ -691,29 +700,27 @@ func TestLeader_ClusterID_upgradePath(t *testing.T) { cleanup func() } - outdated := func(bootstrap bool) server { + outdated := func() server { s, cleanup := TestServer(t, func(c *Config) { c.NumSchedulers = 0 c.Build = before - c.DevDisableBootstrap = bootstrap c.BootstrapExpect = 3 c.Logger.SetLevel(hclog.Trace) }) return server{s: s, cleanup: cleanup} } - upgraded := func(bootstrap bool) server { + upgraded := func() server { s, cleanup := TestServer(t, func(c *Config) { c.NumSchedulers = 0 c.Build = after - c.DevDisableBootstrap = bootstrap - c.BootstrapExpect = 3 + c.BootstrapExpect = 0 c.Logger.SetLevel(hclog.Trace) }) return server{s: s, cleanup: cleanup} } - servers := []server{outdated(false), outdated(true), outdated(true)} + servers := []server{outdated(), outdated(), outdated()} // fallback shutdown attempt in case testing fails defer servers[0].cleanup() defer servers[1].cleanup() @@ -722,7 +729,7 @@ func TestLeader_ClusterID_upgradePath(t *testing.T) { upgrade := func(i int) { previous := servers[i] - servers[i] = upgraded(true) + servers[i] = upgraded() TestJoin(t, servers[i].s, servers[(i+1)%3].s, servers[(i+2)%3].s) testutil.WaitForLeader(t, servers[i].s.RPC) @@ -809,7 +816,6 @@ func TestLeader_ClusterID_noUpgrade(t *testing.T) { c.Logger.SetLevel(hclog.Trace) c.NumSchedulers = 0 c.Build = minClusterIDVersion.String() - c.DevDisableBootstrap = true c.BootstrapExpect = 3 }) defer cleanupS2() @@ -817,7 +823,6 @@ func TestLeader_ClusterID_noUpgrade(t *testing.T) { c.Logger.SetLevel(hclog.Trace) c.NumSchedulers = 0 c.Build = minClusterIDVersion.String() - c.DevDisableBootstrap = true c.BootstrapExpect = 3 }) defer cleanupS3() @@ -1012,13 +1017,13 @@ func TestLeader_UpgradeRaftVersion(t *testing.T) { defer cleanupS1() s2, cleanupS2 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 3 c.RaftConfig.ProtocolVersion = 1 }) defer cleanupS2() s3, cleanupS3 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 3 c.RaftConfig.ProtocolVersion = 2 }) defer cleanupS3() @@ -1054,7 +1059,7 @@ func TestLeader_UpgradeRaftVersion(t *testing.T) { // Replace the dead server with one running raft protocol v3 s4, cleanupS4 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 3 c.Datacenter = "dc1" c.RaftConfig.ProtocolVersion = 3 }) @@ -1111,14 +1116,12 @@ func leaderElectionTest(t *testing.T, raftProtocol raft.ProtocolVersion) { s2, cleanupS2 := TestServer(t, func(c *Config) { c.BootstrapExpect = 3 - c.DevDisableBootstrap = true c.RaftConfig.ProtocolVersion = raftProtocol }) defer cleanupS2() s3, cleanupS3 := TestServer(t, func(c *Config) { c.BootstrapExpect = 3 - c.DevDisableBootstrap = true c.RaftConfig.ProtocolVersion = raftProtocol }) defer cleanupS3() // todo(shoenig) added this, should be here right?? @@ -1170,13 +1173,13 @@ func TestLeader_RollRaftServer(t *testing.T) { defer cleanupS1() s2, cleanupS2 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 3 c.RaftConfig.ProtocolVersion = 2 }) defer cleanupS2() s3, cleanupS3 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 3 c.RaftConfig.ProtocolVersion = 2 }) defer cleanupS3() @@ -1207,7 +1210,7 @@ func TestLeader_RollRaftServer(t *testing.T) { // Replace the dead server with one running raft protocol v3 s4, cleanupS4 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 3 c.RaftConfig.ProtocolVersion = 3 }) defer cleanupS4() @@ -1230,7 +1233,7 @@ func TestLeader_RollRaftServer(t *testing.T) { } // Replace another dead server with one running raft protocol v3 s5, cleanupS5 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 3 c.RaftConfig.ProtocolVersion = 3 }) defer cleanupS5() @@ -1254,7 +1257,7 @@ func TestLeader_RollRaftServer(t *testing.T) { // Replace the last dead server with one running raft protocol v3 s6, cleanupS6 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 3 c.RaftConfig.ProtocolVersion = 3 }) defer cleanupS6() @@ -1330,19 +1333,22 @@ func TestServer_ReconcileMember(t *testing.T) { // Create a three node cluster s1, cleanupS1 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + // disable bootstrapping + c.BootstrapExpect = 0 c.RaftConfig.ProtocolVersion = 3 }) defer cleanupS1() s2, cleanupS2 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + // disable bootstrapping + c.BootstrapExpect = 0 c.RaftConfig.ProtocolVersion = 3 }) defer cleanupS2() s3, cleanupS3 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + // disable bootstrapping + c.BootstrapExpect = 0 c.RaftConfig.ProtocolVersion = 2 }) defer cleanupS3() diff --git a/nomad/node_endpoint_test.go b/nomad/node_endpoint_test.go index a3fb9d5b60d..9d7ebe5ecf8 100644 --- a/nomad/node_endpoint_test.go +++ b/nomad/node_endpoint_test.go @@ -97,7 +97,7 @@ func TestClientEndpoint_Register_NodeConn_Forwarded(t *testing.T) { defer cleanupS1() s2, cleanupS2 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 2 }) defer cleanupS2() TestJoin(t, s1, s2) @@ -754,16 +754,18 @@ func TestClientEndpoint_UpdateStatus_GetEvals(t *testing.T) { func TestClientEndpoint_UpdateStatus_HeartbeatOnly(t *testing.T) { t.Parallel() - s1, cleanupS1 := TestServer(t, nil) + s1, cleanupS1 := TestServer(t, func(c *Config) { + c.BootstrapExpect = 3 + }) defer cleanupS1() s2, cleanupS2 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 3 }) defer cleanupS2() s3, cleanupS3 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 3 }) defer cleanupS3() servers := []*Server{s1, s2, s3} diff --git a/nomad/rpc_test.go b/nomad/rpc_test.go index 4d900c31656..73e47a4f7a5 100644 --- a/nomad/rpc_test.go +++ b/nomad/rpc_test.go @@ -45,10 +45,12 @@ func rpcClient(t *testing.T, s *Server) rpc.ClientCodec { func TestRPC_forwardLeader(t *testing.T) { t.Parallel() - s1, cleanupS1 := TestServer(t, nil) + s1, cleanupS1 := TestServer(t, func(c *Config) { + c.BootstrapExpect = 2 + }) defer cleanupS1() s2, cleanupS2 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 2 }) defer cleanupS2() TestJoin(t, s1, s2) @@ -259,10 +261,12 @@ func TestRPC_streamingRpcConn_badMethod(t *testing.T) { t.Parallel() require := require.New(t) - s1, cleanupS1 := TestServer(t, nil) + s1, cleanupS1 := TestServer(t, func(c *Config) { + c.BootstrapExpect = 2 + }) defer cleanupS1() s2, cleanupS2 := TestServer(t, func(c *Config) { - c.DevDisableBootstrap = true + c.BootstrapExpect = 2 }) defer cleanupS2() TestJoin(t, s1, s2) @@ -298,7 +302,6 @@ func TestRPC_streamingRpcConn_badMethod_TLS(t *testing.T) { c.Region = "regionFoo" c.BootstrapExpect = 2 c.DevMode = false - c.DevDisableBootstrap = true c.DataDir = path.Join(dir, "node1") c.TLSConfig = &config.TLSConfig{ EnableHTTP: true, @@ -315,7 +318,6 @@ func TestRPC_streamingRpcConn_badMethod_TLS(t *testing.T) { c.Region = "regionFoo" c.BootstrapExpect = 2 c.DevMode = false - c.DevDisableBootstrap = true c.DataDir = path.Join(dir, "node2") c.TLSConfig = &config.TLSConfig{ EnableHTTP: true, @@ -354,7 +356,6 @@ func TestRPC_streamingRpcConn_goodMethod_Plaintext(t *testing.T) { c.Region = "regionFoo" c.BootstrapExpect = 2 c.DevMode = false - c.DevDisableBootstrap = true c.DataDir = path.Join(dir, "node1") }) defer cleanupS1() @@ -363,7 +364,6 @@ func TestRPC_streamingRpcConn_goodMethod_Plaintext(t *testing.T) { c.Region = "regionFoo" c.BootstrapExpect = 2 c.DevMode = false - c.DevDisableBootstrap = true c.DataDir = path.Join(dir, "node2") }) defer cleanupS2() @@ -414,7 +414,6 @@ func TestRPC_streamingRpcConn_goodMethod_TLS(t *testing.T) { c.Region = "regionFoo" c.BootstrapExpect = 2 c.DevMode = false - c.DevDisableBootstrap = true c.DataDir = path.Join(dir, "node1") c.TLSConfig = &config.TLSConfig{ EnableHTTP: true, @@ -431,7 +430,6 @@ func TestRPC_streamingRpcConn_goodMethod_TLS(t *testing.T) { c.Region = "regionFoo" c.BootstrapExpect = 2 c.DevMode = false - c.DevDisableBootstrap = true c.DataDir = path.Join(dir, "node2") c.TLSConfig = &config.TLSConfig{ EnableHTTP: true, diff --git a/nomad/serf.go b/nomad/serf.go index 7c00745e3cb..a955e7b1e77 100644 --- a/nomad/serf.go +++ b/nomad/serf.go @@ -83,7 +83,7 @@ func (s *Server) nodeJoin(me serf.MemberEvent) { s.peerLock.Unlock() // If we still expecting to bootstrap, may need to handle this - if atomic.LoadInt32(&s.config.BootstrapExpect) != 0 { + if atomic.LoadInt32(&s.config.Bootstrapped) == 0 { s.maybeBootstrap() } } @@ -111,7 +111,7 @@ func (s *Server) maybeBootstrap() { // Bootstrap can only be done if there are no committed logs, // remove our expectations of bootstrapping if index != 0 { - atomic.StoreInt32(&s.config.BootstrapExpect, 0) + atomic.StoreInt32(&s.config.Bootstrapped, 1) return } @@ -127,7 +127,7 @@ func (s *Server) maybeBootstrap() { if p.Region != s.config.Region { continue } - if p.Expect != 0 && p.Expect != int(atomic.LoadInt32(&s.config.BootstrapExpect)) { + if p.Expect != 0 && p.Expect != s.config.BootstrapExpect { s.logger.Error("peer has a conflicting expect value. All nodes should expect the same number", "member", member) return } @@ -138,11 +138,12 @@ func (s *Server) maybeBootstrap() { if !p.NonVoter { voters++ } + servers = append(servers, *p) } // Skip if we haven't met the minimum expect count - if voters < int(atomic.LoadInt32(&s.config.BootstrapExpect)) { + if voters < s.config.BootstrapExpect { return } @@ -181,7 +182,7 @@ func (s *Server) maybeBootstrap() { if len(peers) > 0 { s.logger.Info("disabling bootstrap mode because existing Raft peers being reported by peer", "peer_name", server.Name, "peer_address", server.Addr) - atomic.StoreInt32(&s.config.BootstrapExpect, 0) + atomic.StoreInt32(&s.config.Bootstrapped, 1) return } } @@ -223,7 +224,7 @@ func (s *Server) maybeBootstrap() { } // Bootstrapping complete, or failed for some reason, don't enter this again - atomic.StoreInt32(&s.config.BootstrapExpect, 0) + atomic.StoreInt32(&s.config.Bootstrapped, 1) } // nodeFailed is used to handle fail events on the serf cluster diff --git a/nomad/serf_test.go b/nomad/serf_test.go index d66dbc38248..56181e9b707 100644 --- a/nomad/serf_test.go +++ b/nomad/serf_test.go @@ -105,7 +105,6 @@ func TestNomad_ReapPeer(t *testing.T) { c.NodeName = "node1" c.BootstrapExpect = 3 c.DevMode = false - c.DevDisableBootstrap = true c.DataDir = path.Join(dir, "node1") }) defer cleanupS1() @@ -113,7 +112,6 @@ func TestNomad_ReapPeer(t *testing.T) { c.NodeName = "node2" c.BootstrapExpect = 3 c.DevMode = false - c.DevDisableBootstrap = true c.DataDir = path.Join(dir, "node2") }) defer cleanupS2() @@ -121,7 +119,6 @@ func TestNomad_ReapPeer(t *testing.T) { c.NodeName = "node3" c.BootstrapExpect = 3 c.DevMode = false - c.DevDisableBootstrap = true c.DataDir = path.Join(dir, "node3") }) defer cleanupS3() @@ -200,21 +197,18 @@ func TestNomad_BootstrapExpect(t *testing.T) { s1, cleanupS1 := TestServer(t, func(c *Config) { c.BootstrapExpect = 3 c.DevMode = false - c.DevDisableBootstrap = true c.DataDir = path.Join(dir, "node1") }) defer cleanupS1() s2, cleanupS2 := TestServer(t, func(c *Config) { c.BootstrapExpect = 3 c.DevMode = false - c.DevDisableBootstrap = true c.DataDir = path.Join(dir, "node2") }) defer cleanupS2() s3, cleanupS3 := TestServer(t, func(c *Config) { c.BootstrapExpect = 3 c.DevMode = false - c.DevDisableBootstrap = true c.DataDir = path.Join(dir, "node3") }) defer cleanupS3() @@ -263,7 +257,6 @@ func TestNomad_BootstrapExpect(t *testing.T) { s4, cleanupS4 := TestServer(t, func(c *Config) { c.BootstrapExpect = 3 c.DevMode = false - c.DevDisableBootstrap = true c.DataDir = path.Join(dir, "node4") }) defer cleanupS4() @@ -313,7 +306,6 @@ func TestNomad_BootstrapExpect_NonVoter(t *testing.T) { s1, cleanupS1 := TestServer(t, func(c *Config) { c.BootstrapExpect = 2 c.DevMode = false - c.DevDisableBootstrap = true c.DataDir = path.Join(dir, "node1") c.NonVoter = true }) @@ -321,7 +313,6 @@ func TestNomad_BootstrapExpect_NonVoter(t *testing.T) { s2, cleanupS2 := TestServer(t, func(c *Config) { c.BootstrapExpect = 2 c.DevMode = false - c.DevDisableBootstrap = true c.DataDir = path.Join(dir, "node2") c.NonVoter = true }) @@ -329,7 +320,6 @@ func TestNomad_BootstrapExpect_NonVoter(t *testing.T) { s3, cleanupS3 := TestServer(t, func(c *Config) { c.BootstrapExpect = 2 c.DevMode = false - c.DevDisableBootstrap = true c.DataDir = path.Join(dir, "node3") }) defer cleanupS3() @@ -351,7 +341,6 @@ func TestNomad_BootstrapExpect_NonVoter(t *testing.T) { s4, cleanupS4 := TestServer(t, func(c *Config) { c.BootstrapExpect = 2 c.DevMode = false - c.DevDisableBootstrap = true c.DataDir = path.Join(dir, "node4") }) defer cleanupS4() @@ -418,12 +407,10 @@ func TestNomad_BadExpect(t *testing.T) { s1, cleanupS1 := TestServer(t, func(c *Config) { c.BootstrapExpect = 2 - c.DevDisableBootstrap = true }) defer cleanupS1() s2, cleanupS2 := TestServer(t, func(c *Config) { c.BootstrapExpect = 3 - c.DevDisableBootstrap = true }) defer cleanupS2() servers := []*Server{s1, s2} @@ -446,7 +433,7 @@ func TestNomad_BadExpect(t *testing.T) { testutil.WaitForResult(func() (bool, error) { for _, s := range servers { p, _ := s.numPeers() - if p != 1 { + if p != 0 { return false, fmt.Errorf("%d", p) } } diff --git a/nomad/server.go b/nomad/server.go index 3ddb4a586ad..0fb95e05d4a 100644 --- a/nomad/server.go +++ b/nomad/server.go @@ -822,7 +822,7 @@ func (s *Server) setupBootstrapHandler() error { // (ab)use serf.go's behavior of setting BootstrapExpect to // zero if we have bootstrapped. If we have bootstrapped - bootstrapExpect := atomic.LoadInt32(&s.config.BootstrapExpect) + bootstrapExpect := s.config.BootstrapExpect if bootstrapExpect == 0 { // This Nomad Server has been bootstrapped. Rely on // the peersTimeout firing as a guard to prevent @@ -848,7 +848,7 @@ func (s *Server) setupBootstrapHandler() error { // quorum has been reached, we do not need to poll // Consul. Let the normal timeout-based strategy // take over. - if raftPeers >= int(bootstrapExpect) { + if raftPeers >= bootstrapExpect { peersTimeout.Reset(peersPollInterval + lib.RandomStagger(peersPollInterval/peersPollJitterFactor)) return nil } @@ -1275,9 +1275,9 @@ func (s *Server) setupRaft() error { } } - // If we are in bootstrap or dev mode and the state is clean then we can + // If we are a single server cluster and the state is clean then we can // bootstrap now. - if s.config.Bootstrap || s.config.DevMode { + if s.isSingleServerCluster() { hasState, err := raft.HasExistingState(log, stable, snap) if err != nil { return err @@ -1320,10 +1320,10 @@ func (s *Server) setupSerf(conf *serf.Config, ch chan serf.Event, path string) ( conf.Tags["id"] = s.config.NodeID conf.Tags["rpc_addr"] = s.clientRpcAdvertise.(*net.TCPAddr).IP.String() // Address that clients will use to RPC to servers conf.Tags["port"] = fmt.Sprintf("%d", s.serverRpcAdvertise.(*net.TCPAddr).Port) // Port servers use to RPC to one and another - if s.config.Bootstrap || (s.config.DevMode && !s.config.DevDisableBootstrap) { + if s.isSingleServerCluster() { conf.Tags["bootstrap"] = "1" } - bootstrapExpect := atomic.LoadInt32(&s.config.BootstrapExpect) + bootstrapExpect := s.config.BootstrapExpect if bootstrapExpect != 0 { conf.Tags["expect"] = fmt.Sprintf("%d", bootstrapExpect) } @@ -1524,7 +1524,7 @@ func (s *Server) Stats() map[string]map[string]string { "server": "true", "leader": fmt.Sprintf("%v", s.IsLeader()), "leader_addr": string(s.raft.Leader()), - "bootstrap": fmt.Sprintf("%v", s.config.Bootstrap), + "bootstrap": fmt.Sprintf("%v", s.isSingleServerCluster()), "known_regions": toString(uint64(len(s.peers))), }, "raft": s.raft.Stats(), @@ -1617,6 +1617,10 @@ func (s *Server) ClusterID() (string, error) { return generatedID, nil } +func (s *Server) isSingleServerCluster() bool { + return s.config.BootstrapExpect == 1 +} + // peersInfoContent is used to help operators understand what happened to the // peers.json file. This is written to a file called peers.info in the same // location. diff --git a/nomad/server_test.go b/nomad/server_test.go index 797b57e62eb..64e34de576b 100644 --- a/nomad/server_test.go +++ b/nomad/server_test.go @@ -56,7 +56,6 @@ func TestServer_RPC_TLS(t *testing.T) { c.Region = "regionFoo" c.BootstrapExpect = 3 c.DevMode = false - c.DevDisableBootstrap = true c.DataDir = path.Join(dir, "node1") c.TLSConfig = &config.TLSConfig{ EnableHTTP: true, @@ -73,7 +72,6 @@ func TestServer_RPC_TLS(t *testing.T) { c.Region = "regionFoo" c.BootstrapExpect = 3 c.DevMode = false - c.DevDisableBootstrap = true c.DataDir = path.Join(dir, "node2") c.TLSConfig = &config.TLSConfig{ EnableHTTP: true, @@ -90,7 +88,6 @@ func TestServer_RPC_TLS(t *testing.T) { c.Region = "regionFoo" c.BootstrapExpect = 3 c.DevMode = false - c.DevDisableBootstrap = true c.DataDir = path.Join(dir, "node3") c.TLSConfig = &config.TLSConfig{ EnableHTTP: true, @@ -125,7 +122,6 @@ func TestServer_RPC_MixedTLS(t *testing.T) { c.Region = "regionFoo" c.BootstrapExpect = 3 c.DevMode = false - c.DevDisableBootstrap = true c.DataDir = path.Join(dir, "node1") c.TLSConfig = &config.TLSConfig{ EnableHTTP: true, @@ -142,7 +138,6 @@ func TestServer_RPC_MixedTLS(t *testing.T) { c.Region = "regionFoo" c.BootstrapExpect = 3 c.DevMode = false - c.DevDisableBootstrap = true c.DataDir = path.Join(dir, "node2") c.TLSConfig = &config.TLSConfig{ EnableHTTP: true, @@ -159,7 +154,6 @@ func TestServer_RPC_MixedTLS(t *testing.T) { c.Region = "regionFoo" c.BootstrapExpect = 3 c.DevMode = false - c.DevDisableBootstrap = true c.DataDir = path.Join(dir, "node3") }) defer cleanupS3() @@ -470,7 +464,6 @@ func TestServer_Reload_TLSConnections_Raft(t *testing.T) { s1, cleanupS1 := TestServer(t, func(c *Config) { c.BootstrapExpect = 2 c.DevMode = false - c.DevDisableBootstrap = true c.DataDir = path.Join(dir, "node1") c.NodeName = "node1" c.Region = "regionFoo" @@ -480,7 +473,6 @@ func TestServer_Reload_TLSConnections_Raft(t *testing.T) { s2, cleanupS2 := TestServer(t, func(c *Config) { c.BootstrapExpect = 2 c.DevMode = false - c.DevDisableBootstrap = true c.DataDir = path.Join(dir, "node2") c.NodeName = "node2" c.Region = "regionFoo" diff --git a/nomad/stats_fetcher_test.go b/nomad/stats_fetcher_test.go index 0278af94299..3c508c73ad3 100644 --- a/nomad/stats_fetcher_test.go +++ b/nomad/stats_fetcher_test.go @@ -13,7 +13,6 @@ func TestStatsFetcher(t *testing.T) { conf := func(c *Config) { c.Region = "region-a" - c.DevDisableBootstrap = true c.BootstrapExpect = 3 } diff --git a/nomad/testing.go b/nomad/testing.go index 8110fb954c1..3beeeb3702c 100644 --- a/nomad/testing.go +++ b/nomad/testing.go @@ -45,6 +45,7 @@ func TestServer(t testing.T, cb func(*Config)) (*Server, func()) { config.Logger = testlog.HCLogger(t) config.Build = version.Version + "+unittest" config.DevMode = true + config.BootstrapExpect = 1 nodeNum := atomic.AddUint32(&nodeNumber, 1) config.NodeName = fmt.Sprintf("nomad-%03d", nodeNum) diff --git a/testutil/wait.go b/testutil/wait.go index 45e23e6b5a9..6ab3ea47211 100644 --- a/testutil/wait.go +++ b/testutil/wait.go @@ -85,6 +85,7 @@ type rpcFn func(string, interface{}, interface{}) error func WaitForLeader(t testing.T, rpc rpcFn) { WaitForResult(func() (bool, error) { args := &structs.GenericRequest{} + args.AllowStale = true var leader string err := rpc("Status.Leader", args, &leader) return leader != "", err