diff --git a/docs/api.md b/docs/api.md index 8cbe22d3..7b255f05 100644 --- a/docs/api.md +++ b/docs/api.md @@ -69,6 +69,7 @@ ClusterSpec defines the desired state for a M3 cluster to be converge to. | podMetadata | PodMetadata is for any Metadata that is unique to the pods, and does not belong on any other objects, such as Prometheus scrape tags | [metav1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#objectmeta-v1-meta) | false | | parallelPodManagement | ParallelPodManagement sets StatefulSets created by the operator to have Parallel pod management instead of OrderedReady. This is an EXPERIMENTAL flag and subject to deprecation in a future release. This has not been tested in production and users should not depend on it without validating it for their own use case. | bool | true | | serviceAccountName | To use a non-default service account, specify the name here otherwise the service account \"default\" will be used. This is useful for advanced use-cases such as pod security policies. The service account must exist. This operator will not create it. | string | false | +| frozen | Frozen is used to stop the operator from taking any further actions on a cluster. This is useful when troubleshooting as it guarantees the operator won't make any changes to the cluster. | bool | false | [Back to TOC](#table-of-contents) diff --git a/go.mod b/go.mod index 9b24fffe..ccaa272c 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,7 @@ require ( github.com/uber-go/tally v3.3.13+incompatible github.com/ultraware/funlen v0.0.2 // indirect github.com/urfave/cli v1.22.2 // indirect + go.uber.org/atomic v1.6.0 go.uber.org/zap v1.13.0 golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f golang.org/x/tools v0.0.0-20200522201501-cb1345f3a375 // indirect diff --git a/pkg/apis/m3dboperator/v1alpha1/cluster.go b/pkg/apis/m3dboperator/v1alpha1/cluster.go index d76b3d56..d10d3609 100644 --- a/pkg/apis/m3dboperator/v1alpha1/cluster.go +++ b/pkg/apis/m3dboperator/v1alpha1/cluster.go @@ -306,6 +306,12 @@ type ClusterSpec struct { // use-cases such as pod security policies. The service account must exist. // This operator will not create it. ServiceAccountName string `json:"serviceAccountName,omitempty"` + + // Frozen is used to stop the operator from taking any further actions on a + // cluster. This is useful when troubleshooting as it guarantees the operator + // won't make any changes to the cluster. + // +optional + Frozen bool `json:"frozen,omitempty"` } // ExternalCoordinatorConfig defines parameters for using an external diff --git a/pkg/apis/m3dboperator/v1alpha1/openapi_generated.go b/pkg/apis/m3dboperator/v1alpha1/openapi_generated.go index 5b012ded..78e2f050 100644 --- a/pkg/apis/m3dboperator/v1alpha1/openapi_generated.go +++ b/pkg/apis/m3dboperator/v1alpha1/openapi_generated.go @@ -617,6 +617,13 @@ func schema_pkg_apis_m3dboperator_v1alpha1_ClusterSpec(ref common.ReferenceCallb Format: "", }, }, + "frozen": { + SchemaProps: spec.SchemaProps{ + Description: "Frozen is used to stop the operator from taking any further actions on a cluster. This is useful when troubleshooting as it guarantees the operator won't make any changes to the cluster.", + Type: []string{"boolean"}, + Format: "", + }, + }, }, Required: []string{"parallelPodManagement"}, }, diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 3cc177a3..383426c3 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -358,6 +358,11 @@ func (c *M3DBController) handleClusterUpdate(cluster *myspec.M3DBCluster) error clusterLogger := c.logger.With(zap.String("cluster", cluster.Name)) + if cluster.Spec.Frozen { + clusterLogger.Info("cluster is frozen so no changes will be made") + return nil + } + // https://v1-12.docs.kubernetes.io/docs/concepts/workloads/controllers/garbage-collection/ // // If deletion timestamp is zero (cluster hasn't been deleted), make sure our diff --git a/pkg/controller/controller_test.go b/pkg/controller/controller_test.go index dc7adea9..0768fc96 100644 --- a/pkg/controller/controller_test.go +++ b/pkg/controller/controller_test.go @@ -57,6 +57,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/uber-go/tally" + "go.uber.org/atomic" "go.uber.org/zap" ) @@ -784,3 +785,36 @@ func TestHandleUpdateClusterUpdatesStatefulSets(t *testing.T) { }) } } + +func TestHandleUpdateClusterFrozen(t *testing.T) { + + var ( + clusterMeta = newMeta("cluster1", map[string]string{ + "foo": "bar", + "operator.m3db.io/app": "m3db", + "operator.m3db.io/cluster": "cluster1", + }, nil) + sets = []*metav1.ObjectMeta{ + newMeta("cluster1-rep0", nil, nil), + } + ) + cluster, deps := setupTestCluster(t, *clusterMeta, sets, 3) + defer deps.cleanup() + controller := deps.newController(t) + + cluster.Spec.Frozen = true + + count := atomic.NewInt64(0) + controller.kubeClient.(*kubefake.Clientset).PrependReactor( + "*", "*", func(action ktesting.Action) (bool, runtime.Object, error) { + count.Inc() + return false, nil, nil + }) + + for i := 0; i < 20; i++ { + err := controller.handleClusterUpdate(cluster) + require.NoError(t, err) + } + + assert.Equal(t, int64(0), count.Load()) +}