[dbnode] Fix node serving reads/marked bootstrapped after bootstrap f…

…ailure (#3088)

* set bootstrap state to Bootstrapped when it returns no errors

* added bootstrap regression test.

* removed new test and updated old test to check for bootstrap states.

src/dbnode/storage/bootstrap.go

@@ -153,8 +153,6 @@ func (m *bootstrapManager) Bootstrap() (BootstrapResult, error) {
 		if currPending {
 			// New bootstrap calls should now enqueue another pending bootstrap
 			m.hasPending = false
-		} else {
-			m.state = Bootstrapped
 		}
 		m.Unlock()
 
@@ -186,6 +184,7 @@ func (m *bootstrapManager) Bootstrap() (BootstrapResult, error) {
 	// across the cluster.
 	m.Lock()
 	m.lastBootstrapCompletionTime = xtime.ToUnixNano(m.nowFn())
+	m.state = Bootstrapped
 	m.Unlock()
 	return result, nil
 }

src/dbnode/storage/bootstrap_test.go

@@ -32,9 +32,10 @@ import (
 	"github.com/m3db/m3/src/x/ident"
 
 	"github.com/golang/mock/gomock"
-	xtest "github.com/m3db/m3/src/x/test"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	xtest "github.com/m3db/m3/src/x/test"
 )
 
 func TestDatabaseBootstrapWithBootstrapError(t *testing.T) {
@@ -74,7 +75,9 @@ func TestDatabaseBootstrapWithBootstrapError(t *testing.T) {
 			Return(fmt.Errorf("an error")).
 			Do(func(ctx context.Context, bootstrapResult bootstrap.NamespaceResult) {
 				// After returning an error, make sure we don't re-enqueue.
+				require.Equal(t, Bootstrapping, bsm.state)
 				bsm.bootstrapFn = func() error {
+					require.Equal(t, Bootstrapping, bsm.state)
 					return nil
 				}
 			}),
@@ -83,9 +86,12 @@ func TestDatabaseBootstrapWithBootstrapError(t *testing.T) {
 	ctx := context.NewContext()
 	defer ctx.Close()
 
+	require.Equal(t, BootstrapNotStarted, bsm.state)
+
 	result, err := bsm.Bootstrap()
-	require.NoError(t, err)
 
+	require.NoError(t, err)
+	require.Equal(t, Bootstrapped, bsm.state)
 	require.Equal(t, 1, len(result.ErrorsBootstrap))
 	require.Equal(t, "an error", result.ErrorsBootstrap[0].Error())
 }