Use Feature Gates to enable or disable features (#847)

There are a number of features that are being developed and likely to be
disabled by default in their early stage.

Instead of adding a temporary config for each feature and maintaining
them separately, this patch introduces Feature Gates to toggle the
features. It will be easier to choose code branch based on FeatureGates'
"Enabled" method and to promote features to beta and GA.

build/yamls/antrea-eks.yml

@@ -344,6 +344,9 @@ metadata:
 apiVersion: v1
 data:
   antrea-agent.conf: |
+    # FeatureGates is a map of feature names to bools that enable or disable experimental features.
+    #featureGates:
+
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     #ovsBridge: br-int
@@ -413,6 +416,9 @@ data:
         ]
     }
   antrea-controller.conf: |
+    # FeatureGates is a map of feature names to bools that enable or disable experimental features.
+    #featureGates:
+
     # The port for the antrea-controller APIServer to serve on.
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-controller` container must be set to the same value.
@@ -432,7 +438,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-hmd2mdhg89
+  name: antrea-config-2k49hdb86m
   namespace: kube-system
 ---
 apiVersion: v1
@@ -537,7 +543,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-hmd2mdhg89
+          name: antrea-config-2k49hdb86m
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -751,7 +757,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-hmd2mdhg89
+          name: antrea-config-2k49hdb86m
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

build/yamls/antrea-gke.yml

@@ -344,6 +344,9 @@ metadata:
 apiVersion: v1
 data:
   antrea-agent.conf: |
+    # FeatureGates is a map of feature names to bools that enable or disable experimental features.
+    #featureGates:
+
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     #ovsBridge: br-int
@@ -413,6 +416,9 @@ data:
         ]
     }
   antrea-controller.conf: |
+    # FeatureGates is a map of feature names to bools that enable or disable experimental features.
+    #featureGates:
+
     # The port for the antrea-controller APIServer to serve on.
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-controller` container must be set to the same value.
@@ -432,7 +438,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-ff5ff2btgc
+  name: antrea-config-cfhgb2tt48
   namespace: kube-system
 ---
 apiVersion: v1
@@ -537,7 +543,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-ff5ff2btgc
+          name: antrea-config-cfhgb2tt48
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -749,7 +755,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-ff5ff2btgc
+          name: antrea-config-cfhgb2tt48
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

build/yamls/antrea-ipsec.yml

@@ -344,6 +344,9 @@ metadata:
 apiVersion: v1
 data:
   antrea-agent.conf: |
+    # FeatureGates is a map of feature names to bools that enable or disable experimental features.
+    #featureGates:
+
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     #ovsBridge: br-int
@@ -413,6 +416,9 @@ data:
         ]
     }
   antrea-controller.conf: |
+    # FeatureGates is a map of feature names to bools that enable or disable experimental features.
+    #featureGates:
+
     # The port for the antrea-controller APIServer to serve on.
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-controller` container must be set to the same value.
@@ -432,7 +438,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-fggkd66d2h
+  name: antrea-config-57kg4gbmk6
   namespace: kube-system
 ---
 apiVersion: v1
@@ -546,7 +552,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-fggkd66d2h
+          name: antrea-config-57kg4gbmk6
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -793,7 +799,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-fggkd66d2h
+          name: antrea-config-57kg4gbmk6
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

build/yamls/antrea.yml

@@ -344,6 +344,9 @@ metadata:
 apiVersion: v1
 data:
   antrea-agent.conf: |
+    # FeatureGates is a map of feature names to bools that enable or disable experimental features.
+    #featureGates:
+
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     #ovsBridge: br-int
@@ -413,6 +416,9 @@ data:
         ]
     }
   antrea-controller.conf: |
+    # FeatureGates is a map of feature names to bools that enable or disable experimental features.
+    #featureGates:
+
     # The port for the antrea-controller APIServer to serve on.
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-controller` container must be set to the same value.
@@ -432,7 +438,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-mf4t8c67c8
+  name: antrea-config-cd7tt4t2f8
   namespace: kube-system
 ---
 apiVersion: v1
@@ -537,7 +543,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-mf4t8c67c8
+          name: antrea-config-cd7tt4t2f8
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -749,7 +755,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-mf4t8c67c8
+          name: antrea-config-cd7tt4t2f8
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

build/yamls/base/conf/antrea-agent.conf

@@ -1,3 +1,6 @@
+# FeatureGates is a map of feature names to bools that enable or disable experimental features.
+#featureGates:
+
 # Name of the OpenVSwitch bridge antrea-agent will create and use.
 # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
 #ovsBridge: br-int

build/yamls/base/conf/antrea-controller.conf

@@ -1,3 +1,6 @@
+# FeatureGates is a map of feature names to bools that enable or disable experimental features.
+#featureGates:
+
 # The port for the antrea-controller APIServer to serve on.
 # Note that if it's set to another value, the `containerPort` of the `api` port of the
 # `antrea-controller` container must be set to the same value.

cmd/antrea-agent/config.go

@@ -19,6 +19,9 @@ import (
 )
 
 type AgentConfig struct {
+	// featureGates is a map of feature names to bools that enable or disable experimental features.
+	FeatureGates map[string]bool `yaml:"featureGates,omitempty"`
+
 	CNISocket string `yaml:"cniSocket,omitempty"`
 	// clientConnection specifies the kubeconfig file and client connection settings for the agent
 	// to communicate with the apiserver.

cmd/antrea-agent/options.go

@@ -25,6 +25,7 @@ import (
 	"github.com/vmware-tanzu/antrea/pkg/agent/config"
 	"github.com/vmware-tanzu/antrea/pkg/apis"
 	"github.com/vmware-tanzu/antrea/pkg/cni"
+	"github.com/vmware-tanzu/antrea/pkg/features"
 	"github.com/vmware-tanzu/antrea/pkg/ovs/ovsconfig"
 )
 
@@ -71,7 +72,7 @@ func (o *Options) complete(args []string) error {
 		o.config = c
 	}
 	o.setDefaults()
-	return nil
+	return features.DefaultMutableFeatureGate.SetFromMap(o.config.FeatureGates)
 }
 
 // validate validates all the required options. It must be called after complete.

cmd/antrea-controller/config.go

@@ -19,6 +19,8 @@ import (
 )
 
 type ControllerConfig struct {
+	// FeatureGates is a map of feature names to bools that enable or disable experimental features.
+	FeatureGates map[string]bool `yaml:"featureGates,omitempty"`
 	// clientConnection specifies the kubeconfig file and client connection settings for the
 	// antrea-controller to communicate with the Kubernetes apiserver.
 	ClientConnection componentbaseconfig.ClientConnectionConfiguration `yaml:"clientConnection"`
@@ -35,8 +37,4 @@ type ControllerConfig struct {
 	//   tls.key: <TLS private key>
 	// Defaults to true.
 	SelfSignedCert bool `yaml:"selfSignedCert,omitempty"`
-	// Enable controller to watch for ClusterNetworkPolicy CRDs. Temporary config option
-	// to be removed once CRDs are considered stable.
-	// Defaults to false.
-	EnableSecurityCRDs bool `yaml:"enableSecurityCRDs,omitempty"`
 }

cmd/antrea-controller/controller.go

@@ -39,6 +39,7 @@ import (
 	"github.com/vmware-tanzu/antrea/pkg/controller/networkpolicy"
 	"github.com/vmware-tanzu/antrea/pkg/controller/networkpolicy/store"
 	"github.com/vmware-tanzu/antrea/pkg/controller/querier"
+	"github.com/vmware-tanzu/antrea/pkg/features"
 	"github.com/vmware-tanzu/antrea/pkg/k8s"
 	"github.com/vmware-tanzu/antrea/pkg/monitor"
 	"github.com/vmware-tanzu/antrea/pkg/signals"
@@ -111,8 +112,8 @@ func run(o *Options) error {
 	stopCh := signals.RegisterSignalHandlers()
 
 	informerFactory.Start(stopCh)
-	// Only start watching Security CRDs when config option is set to true.
-	if o.config.EnableSecurityCRDs {
+	// Only start watching Security CRDs when ClusterNetworkPolicy is enabled.
+	if features.DefaultFeatureGate.Enabled(features.ClusterNetworkPolicy) {
 		crdInformerFactory.Start(stopCh)
 	}
 

cmd/antrea-controller/options.go

@@ -22,6 +22,7 @@ import (
 	"gopkg.in/yaml.v2"
 
 	"github.com/vmware-tanzu/antrea/pkg/apis"
+	"github.com/vmware-tanzu/antrea/pkg/features"
 )
 
 type Options struct {
@@ -52,7 +53,7 @@ func (o *Options) complete(args []string) error {
 		o.config = c
 	}
 	o.setDefaults()
-	return nil
+	return features.DefaultMutableFeatureGate.SetFromMap(o.config.FeatureGates)
 }
 
 // validate validates all the required options.

pkg/features/antrea_features.go

@@ -0,0 +1,52 @@
+// Copyright 2020 Antrea Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package features
+
+import (
+	"k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/component-base/featuregate"
+)
+
+const (
+	// Every feature gate should add constant here following this template:
+	//
+	// alpha: vX.Y
+	// beta: vX.Y
+	// MyFeature featuregate.Feature = "MyFeature"
+
+	// alpha: v0.8
+	// Allows to apply cluster-wide NetworkPolicies.
+	ClusterNetworkPolicy featuregate.Feature = "ClusterNetworkPolicy"
+)
+
+var (
+	// DefaultMutableFeatureGate is a mutable version of DefaultFeatureGate.
+	DefaultMutableFeatureGate featuregate.MutableFeatureGate = featuregate.NewFeatureGate()
+
+	// DefaultFeatureGate is a shared global FeatureGate.
+	// The feature gate should be modified via DefaultMutableFeatureGate.
+	DefaultFeatureGate featuregate.FeatureGate = DefaultMutableFeatureGate
+
+	// defaultAntreaFeatureGates consists of all known Antrea-specific feature keys.
+	// To add a new feature, define a key for it above and add it here. The features will be
+	// available throughout Antrea binaries.
+	defaultAntreaFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{
+		ClusterNetworkPolicy: {Default: false, PreRelease: featuregate.Alpha},
+	}
+)
+
+func init() {
+	runtime.Must(DefaultMutableFeatureGate.Add(defaultAntreaFeatureGates))
+}