Merge pull request #4451 from elmiko/expand-capi-group

expand CAPI_GROUP usage to cover other capi group variables

cluster-autoscaler/cloudprovider/clusterapi/README.md

@@ -121,6 +121,15 @@ some situations, such as testing or prototyping, you may wish to change this
 group variable. For these situations you may use the environment variable
 `CAPI_GROUP` to change the group that the provider will use.
 
+Please note that setting the `CAPI_GROUP` environment variable will also cause the
+annotations for minimum and maximum size to change.
+This behavior will also affect the machine annotation on nodes, the machine deletion annotation,
+and the cluster name label. For example, if `CAPI_GROUP=test.k8s.io`
+then the minimum size annotation key will be `test.k8s.io/cluster-api-autoscaler-node-group-min-size`,
+the machine annotation on nodes will be `test.8s.io/machine`, the machine deletion
+annotation will be `test.k8s.io/delete-machine`, and the cluster name label will be
+`test.k8s.io/cluster-name`.
+
 ## Sample manifest
 
 A sample manifest that will create a deployment running the autoscaler is

cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup.go

@@ -32,8 +32,6 @@ const (
 	deprecatedMachineDeleteAnnotationKey = "cluster.k8s.io/delete-machine"
 	// TODO: determine what currently relies on deprecatedMachineAnnotationKey to determine when it can be removed
 	deprecatedMachineAnnotationKey = "cluster.k8s.io/machine"
-	machineDeleteAnnotationKey     = "cluster.x-k8s.io/delete-machine"
-	machineAnnotationKey           = "cluster.x-k8s.io/machine"
 	debugFormat                    = "%s (min: %d, max: %d, replicas: %d)"
 )
 

cluster-autoscaler/cloudprovider/clusterapi/clusterapi_utils.go

@@ -17,6 +17,7 @@ limitations under the License.
 package clusterapi
 
 import (
+	"fmt"
 	"strconv"
 	"strings"
 
@@ -28,13 +29,15 @@ import (
 const (
 	deprecatedNodeGroupMinSizeAnnotationKey = "cluster.k8s.io/cluster-api-autoscaler-node-group-min-size"
 	deprecatedNodeGroupMaxSizeAnnotationKey = "cluster.k8s.io/cluster-api-autoscaler-node-group-max-size"
-	nodeGroupMinSizeAnnotationKey           = "cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size"
-	nodeGroupMaxSizeAnnotationKey           = "cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size"
-	clusterNameLabel                        = "cluster.x-k8s.io/cluster-name"
 	deprecatedClusterNameLabel              = "cluster.k8s.io/cluster-name"
 )
 
 var (
+	// clusterNameLabel is the label applied to objects(Machine, MachineSet, MachineDeployment)
+	// to identify which cluster they are owned by. Because the label can be
+	// affected by the CAPI_GROUP environment variable, it is initialized here.
+	clusterNameLabel = getClusterNameLabel()
+
 	// errMissingMinAnnotation is the error returned when a
 	// machine set does not have an annotation keyed by
 	// nodeGroupMinSizeAnnotationKey.
@@ -52,6 +55,23 @@ var (
 	// errInvalidMaxAnnotationValue is the error returned when a
 	// machine set has a non-integral max annotation value.
 	errInvalidMaxAnnotation = errors.New("invalid max annotation")
+
+	// machineDeleteAnnotationKey is the annotation used by cluster-api to indicate
+	// that a machine should be deleted. Because this key can be affected by the
+	// CAPI_GROUP env variable, it is initialized here.
+	machineDeleteAnnotationKey = getMachineDeleteAnnotationKey()
+
+	// machineAnnotationKey is the annotation used by the cluster-api on Node objects
+	// to specify the name of the related Machine object. Because this can be affected
+	// by the CAPI_GROUP env variable, it is initialized here.
+	machineAnnotationKey = getMachineAnnotationKey()
+
+	// nodeGroupMinSizeAnnotationKey and nodeGroupMaxSizeAnnotationKey are the keys
+	// used in MachineSet and MachineDeployment annotations to specify the limits
+	// for the node group. Because the keys can be affected by the CAPI_GROUP env
+	// variable, they are initialized here.
+	nodeGroupMinSizeAnnotationKey = getNodeGroupMinSizeAnnotationKey()
+	nodeGroupMaxSizeAnnotationKey = getNodeGroupMaxSizeAnnotationKey()
 )
 
 type normalizedProviderID string
@@ -180,3 +200,43 @@ func clusterNameFromResource(r *unstructured.Unstructured) string {
 
 	return ""
 }
+
+// getNodeGroupMinSizeAnnotationKey returns the key that is used for the
+// node group minimum size annotation. This function is needed because the user can
+// change the default group name by using the CAPI_GROUP environment variable.
+func getNodeGroupMinSizeAnnotationKey() string {
+	key := fmt.Sprintf("%s/cluster-api-autoscaler-node-group-min-size", getCAPIGroup())
+	return key
+}
+
+// getNodeGroupMaxSizeAnnotationKey returns the key that is used for the
+// node group maximum size annotation. This function is needed because the user can
+// change the default group name by using the CAPI_GROUP environment variable.
+func getNodeGroupMaxSizeAnnotationKey() string {
+	key := fmt.Sprintf("%s/cluster-api-autoscaler-node-group-max-size", getCAPIGroup())
+	return key
+}
+
+// getMachineDeleteAnnotationKey returns the key that is used by cluster-api for marking
+// machines to be deleted. This function is needed because the user can change the default
+// group name by using the CAPI_GROUP environment variable.
+func getMachineDeleteAnnotationKey() string {
+	key := fmt.Sprintf("%s/delete-machine", getCAPIGroup())
+	return key
+}
+
+// getMachineAnnotationKey returns the key that is used by cluster-api for annotating
+// nodes with their related machine objects. This function is needed because the user can change
+// the default group name by using the CAPI_GROUP environment variable.
+func getMachineAnnotationKey() string {
+	key := fmt.Sprintf("%s/machine", getCAPIGroup())
+	return key
+}
+
+// getClusterNameLabel returns the key that is used by cluster-api for labeling
+// which cluster an object belongs to. This function is needed because the user can change
+// the default group name by using the CAPI_GROUP environment variable.
+func getClusterNameLabel() string {
+	key := fmt.Sprintf("%s/cluster-name", getCAPIGroup())
+	return key
+}

cluster-autoscaler/cloudprovider/clusterapi/clusterapi_utils_test.go

@@ -17,6 +17,8 @@ limitations under the License.
 package clusterapi
 
 import (
+	"fmt"
+	"os"
 	"reflect"
 	"strings"
 	"testing"
@@ -689,4 +691,90 @@ func Test_clusterNameFromResource(t *testing.T) {
 			}
 		})
 	}
+
+}
+
+func Test_getKeyHelpers(t *testing.T) {
+	for _, tc := range []struct {
+		name     string
+		expected string
+		testfunc func() string
+	}{
+		{
+			name:     "default group, min size annotation key",
+			expected: fmt.Sprintf("%s/cluster-api-autoscaler-node-group-min-size", defaultCAPIGroup),
+			testfunc: getNodeGroupMinSizeAnnotationKey,
+		},
+		{
+			name:     "default group, max size annotation key",
+			expected: fmt.Sprintf("%s/cluster-api-autoscaler-node-group-max-size", defaultCAPIGroup),
+			testfunc: getNodeGroupMaxSizeAnnotationKey,
+		},
+		{
+			name:     "default group, machine delete annotation key",
+			expected: fmt.Sprintf("%s/delete-machine", defaultCAPIGroup),
+			testfunc: getMachineDeleteAnnotationKey,
+		},
+		{
+			name:     "default group, machine annotation key",
+			expected: fmt.Sprintf("%s/machine", defaultCAPIGroup),
+			testfunc: getMachineAnnotationKey,
+		},
+		{
+			name:     "default group, cluster name label",
+			expected: fmt.Sprintf("%s/cluster-name", defaultCAPIGroup),
+			testfunc: getClusterNameLabel,
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			observed := tc.testfunc()
+			if observed != tc.expected {
+				t.Errorf("%s, mismatch, expected=%s, observed=%s", tc.name, observed, tc.expected)
+			}
+		})
+	}
+
+	testgroup := "test.k8s.io"
+	if err := os.Setenv(CAPIGroupEnvVar, testgroup); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	for _, tc := range []struct {
+		name     string
+		expected string
+		testfunc func() string
+	}{
+		{
+			name:     "test group, min size annotation key",
+			expected: fmt.Sprintf("%s/cluster-api-autoscaler-node-group-min-size", testgroup),
+			testfunc: getNodeGroupMinSizeAnnotationKey,
+		},
+		{
+			name:     "test group, max size annotation key",
+			expected: fmt.Sprintf("%s/cluster-api-autoscaler-node-group-max-size", testgroup),
+			testfunc: getNodeGroupMaxSizeAnnotationKey,
+		},
+		{
+			name:     "test group, machine delete annotation key",
+			expected: fmt.Sprintf("%s/delete-machine", testgroup),
+			testfunc: getMachineDeleteAnnotationKey,
+		},
+		{
+			name:     "test group, machine annotation key",
+			expected: fmt.Sprintf("%s/machine", testgroup),
+			testfunc: getMachineAnnotationKey,
+		},
+		{
+			name:     "test group, cluster name label",
+			expected: fmt.Sprintf("%s/cluster-name", testgroup),
+			testfunc: getClusterNameLabel,
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			observed := tc.testfunc()
+			if observed != tc.expected {
+				t.Errorf("%s, mismatch, expected=%s, observed=%s", tc.name, observed, tc.expected)
+			}
+		})
+	}
 }