add recommended policies feature to operator

Signed-off-by: rksharma95 <[email protected]>

deployments/helm/KubeArmorOperator/crds/operator.kubearmor.com_kubearmorconfigs.yaml

@@ -127,6 +127,31 @@ spec:
                     - Never
                     type: string
                 type: object
+              maxAlertPerSec:
+                type: integer
+              recommendedPolicies:
+                properties:
+                  enable:
+                    type: boolean
+                  matchExpressions:
+                    items:
+                      properties:
+                        key:
+                          enum:
+                          - namespace
+                          type: string
+                        operator:
+                          enum:
+                          - In
+                          - NotIn
+                          type: string
+                        values:
+                          items:
+                            type: string
+                          type: array
+                      type: object
+                    type: array
+                type: object
               seccompEnabled:
                 type: boolean
               alertThrottling:

deployments/operator/operator.yaml

@@ -126,6 +126,31 @@ spec:
                     - Never
                     type: string
                 type: object
+              maxAlertPerSec:
+                type: integer
+              recommendedPolicies:
+                properties:
+                  enable:
+                    type: boolean
+                  matchExpressions:
+                    items:
+                      properties:
+                        key:
+                          enum:
+                          - namespace
+                          type: string
+                        operator:
+                          enum:
+                          - In
+                          - NotIn
+                          type: string
+                        values:
+                          items:
+                            type: string
+                          type: array
+                      type: object
+                    type: array
+                type: object
               seccompEnabled:
                 type: boolean
               alertThrottling:

pkg/KubeArmorOperator/Dockerfile

@@ -35,6 +35,7 @@ COPY $OPERATOR_DIR/enforcer enforcer
 COPY $OPERATOR_DIR/k8s k8s
 COPY $OPERATOR_DIR/runtime runtime
 COPY $OPERATOR_DIR/seccomp seccomp
+COPY $OPERATOR_DIR/recommend recommend
 
 # Build
 RUN CGO_ENABLED=0 GOOS=${GOOS} GOARCH=${GOARCH} GO111MODULE=on go build -a -o operator cmd/operator/main.go

pkg/KubeArmorOperator/api/operator.kubearmor.com/v1/kubearmorconfig_types.go

@@ -4,6 +4,7 @@
 package v1
 
 import (
+	securityv1 "github.com/kubearmor/KubeArmor/pkg/KubeArmorController/api/security.kubearmor.com/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -30,11 +31,19 @@ type Tls struct {
 	RelayExtraIpAddresses []string `json:"extraIpAddresses,omitempty"`
 }
 
+type RecommendedPolicies struct {
+	Enable bool `json:"enable,omitempty"`
+
+	MatchExpressions []securityv1.MatchExpressionsType `json:"matchExpressions,omitempty"`
+}
+
 // KubeArmorConfigSpec defines the desired state of KubeArmorConfig
 type KubeArmorConfigSpec struct {
 	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
 	// Important: Run "make" to regenerate code after modifying this file
 
+	// +kubebuilder:validation:optional
+	RecommendedPolicies RecommendedPolicies `json:"recommendedPolicies,omitempty"`
 	// +kubebuilder:validation:optional
 	DefaultFilePosture PostureType `json:"defaultFilePosture,omitempty"`
 	// +kubebuilder:validation:optional

pkg/KubeArmorOperator/cmd/operator/main.go

@@ -8,6 +8,7 @@ import (
 	"errors"
 	"path/filepath"
 
+	secv1client "github.com/kubearmor/KubeArmor/pkg/KubeArmorController/client/clientset/versioned"
 	opv1client "github.com/kubearmor/KubeArmor/pkg/KubeArmorOperator/client/clientset/versioned"
 	controllers "github.com/kubearmor/KubeArmor/pkg/KubeArmorOperator/internal/controller"
 	"github.com/kubearmor/KubeArmor/pkg/KubeArmorOperator/k8s"
@@ -28,6 +29,7 @@ var PathPrefix string
 var DeploymentName string
 var ExtClient *apiextensionsclientset.Clientset
 var Opv1Client *opv1client.Clientset
+var Secv1Client *secv1client.Clientset
 var InitDeploy bool
 var LogLevel string
 
@@ -45,14 +47,15 @@ var Cmd = &cobra.Command{
 		K8sClient = k8s.NewClient(*Logger, KubeConfig)
 		ExtClient = k8s.NewExtClient(*Logger, KubeConfig)
 		Opv1Client = k8s.NewOpv1Client(*Logger, KubeConfig)
+		Secv1Client = k8s.NewSecv1Client(*Logger, KubeConfig)
 		//Initialise k8sClient for all child commands to inherit
 		if K8sClient == nil {
 			return errors.New("couldn't create k8s client")
 		}
 		return nil
 	},
 	Run: func(cmd *cobra.Command, args []string) {
-		nodeWatcher := controllers.NewClusterWatcher(K8sClient, Logger, ExtClient, Opv1Client, PathPrefix, DeploymentName, InitDeploy)
+		nodeWatcher := controllers.NewClusterWatcher(K8sClient, Logger, ExtClient, Opv1Client, Secv1Client, PathPrefix, DeploymentName, InitDeploy)
 		go nodeWatcher.WatchConfigCrd()
 		nodeWatcher.WatchNodes()
 

pkg/KubeArmorOperator/common/defaults.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 
 	deployments "github.com/kubearmor/KubeArmor/deployments/get"
+	securityv1 "github.com/kubearmor/KubeArmor/pkg/KubeArmorController/api/security.kubearmor.com/v1"
 	opv1 "github.com/kubearmor/KubeArmor/pkg/KubeArmorOperator/api/operator.kubearmor.com/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -124,6 +125,20 @@ var (
 	AlertThrottling       bool   = true
 	DefaultMaxAlertPerSec string = "10"
 	DefaultThrottleSec    string = "30"
+
+	// recommend policies
+	RecommendedPolicies opv1.RecommendedPolicies = opv1.RecommendedPolicies{
+		MatchExpressions: []securityv1.MatchExpressionsType{
+			{
+				Key:      "namespace",
+				Operator: "NotIn",
+				Values: []string{
+					"kube-system",
+					"kubearmor",
+				},
+			},
+		},
+	}
 )
 
 var ConfigMapData = map[string]string{

pkg/KubeArmorOperator/config/crd/bases/operator.kubearmor.com_kubearmorconfigs.yaml

@@ -128,6 +128,31 @@ spec:
                     - Never
                     type: string
                 type: object
+              maxAlertPerSec:
+                type: integer
+              recommendedPolicies:
+                properties:
+                  enable:
+                    type: boolean
+                  matchExpressions:
+                    items:
+                      properties:
+                        key:
+                          enum:
+                          - namespace
+                          type: string
+                        operator:
+                          enum:
+                          - In
+                          - NotIn
+                          type: string
+                        values:
+                          items:
+                            type: string
+                          type: array
+                      type: object
+                    type: array
+                type: object
               seccompEnabled:
                 type: boolean
               alertThrottling:

pkg/KubeArmorOperator/internal/controller/cluster.go

@@ -7,24 +7,30 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"reflect"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
 	certutil "github.com/kubearmor/KubeArmor/KubeArmor/cert"
 	deployments "github.com/kubearmor/KubeArmor/deployments/get"
+	secv1 "github.com/kubearmor/KubeArmor/pkg/KubeArmorController/api/security.kubearmor.com/v1"
+	secv1client "github.com/kubearmor/KubeArmor/pkg/KubeArmorController/client/clientset/versioned"
 	opv1 "github.com/kubearmor/KubeArmor/pkg/KubeArmorOperator/api/operator.kubearmor.com/v1"
 	"github.com/kubearmor/KubeArmor/pkg/KubeArmorOperator/cert"
 	opv1client "github.com/kubearmor/KubeArmor/pkg/KubeArmorOperator/client/clientset/versioned"
+	"github.com/kubearmor/KubeArmor/pkg/KubeArmorOperator/client/clientset/versioned/scheme"
 	opv1Informer "github.com/kubearmor/KubeArmor/pkg/KubeArmorOperator/client/informers/externalversions"
 	"github.com/kubearmor/KubeArmor/pkg/KubeArmorOperator/common"
+	"github.com/kubearmor/KubeArmor/pkg/KubeArmorOperator/recommend"
 	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
 	apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	metav1errors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/informers"
@@ -46,6 +52,7 @@ type ClusterWatcher struct {
 	Client         *kubernetes.Clientset
 	ExtClient      *apiextensionsclientset.Clientset
 	Opv1Client     *opv1client.Clientset
+	Secv1Client    *secv1client.Clientset
 	Daemonsets     map[string]int
 	DaemonsetsLock *sync.Mutex
 }
@@ -60,7 +67,7 @@ type Node struct {
 	Seccomp       string
 }
 
-func NewClusterWatcher(client *kubernetes.Clientset, log *zap.SugaredLogger, extClient *apiextensionsclientset.Clientset, opv1Client *opv1client.Clientset, pathPrefix, deploy_name string, initdeploy bool) *ClusterWatcher {
+func NewClusterWatcher(client *kubernetes.Clientset, log *zap.SugaredLogger, extClient *apiextensionsclientset.Clientset, opv1Client *opv1client.Clientset, secv1Client *secv1client.Clientset, pathPrefix, deploy_name string, initdeploy bool) *ClusterWatcher {
 	if informer == nil {
 		informer = informers.NewSharedInformerFactory(client, 0)
 	}
@@ -86,6 +93,7 @@ func NewClusterWatcher(client *kubernetes.Clientset, log *zap.SugaredLogger, ext
 		Client:         client,
 		ExtClient:      extClient,
 		Opv1Client:     opv1Client,
+		Secv1Client:    secv1Client,
 	}
 }
 
@@ -299,6 +307,7 @@ func (clusterWatcher *ClusterWatcher) WatchConfigCrd() {
 						UpdateImages(&cfg.Spec)
 						UpdatedKubearmorRelayEnv(&cfg.Spec)
 						UpdatedSeccomp(&cfg.Spec)
+						UpdateRecommendedPolicyConfig(&cfg.Spec)
 						// update status to (Installation) Created
 						go clusterWatcher.UpdateCrdStatus(cfg.Name, common.CREATED, common.CREATED_MSG)
 						go clusterWatcher.WatchRequiredResources()
@@ -322,6 +331,7 @@ func (clusterWatcher *ClusterWatcher) WatchConfigCrd() {
 						relayEnvUpdated := UpdatedKubearmorRelayEnv(&cfg.Spec)
 						seccompEnabledUpdated := UpdatedSeccomp(&cfg.Spec)
 						tlsUpdated := UpdateTlsData(&cfg.Spec)
+						UpdateRecommendedPolicyConfig(&cfg.Spec)
 						// return if only status has been updated
 						if !tlsUpdated && !relayEnvUpdated && !configChanged && cfg.Status != oldObj.(*opv1.KubeArmorConfig).Status && len(imageUpdated) < 1 {
 							return
@@ -779,6 +789,99 @@ func (clusterWatcher *ClusterWatcher) UpdateTlsConfigurations(tlsEnabled bool) e
 	return nil
 }
 
+func (clusterWatcher *ClusterWatcher) WatchRecommendedPolicies() error {
+	switch common.RecommendedPolicies.Enable {
+	case true:
+		policies, err := recommend.CRDFs.ReadDir(".")
+		if err != nil {
+			clusterWatcher.Log.Warnf("error reading policies FS", err)
+			return err
+		}
+		for _, policy := range policies {
+			if !policy.IsDir() {
+				yamlBytes, err := recommend.CRDFs.ReadFile(policy.Name())
+				if err != nil {
+					clusterWatcher.Log.Warnf("error reading csp", policy.Name())
+					continue
+				}
+				csp := &secv1.KubeArmorClusterPolicy{}
+				if err := runtime.DecodeInto(scheme.Codecs.UniversalDeserializer(), yamlBytes, csp); err != nil {
+					clusterWatcher.Log.Warnf("error decoding csp", policy.Name())
+					continue
+				}
+				csp.Spec.Selector.MatchExpressions = common.RecommendedPolicies.MatchExpressions
+				_, err = clusterWatcher.Secv1Client.SecurityV1().KubeArmorClusterPolicies().Create(context.Background(), csp, metav1.CreateOptions{})
+				if err != nil && !metav1errors.IsAlreadyExists(err) {
+					clusterWatcher.Log.Warnf("error creating csp", csp.GetName())
+					continue
+				} else if metav1errors.IsAlreadyExists(err) {
+					pol, err := clusterWatcher.Secv1Client.SecurityV1().KubeArmorClusterPolicies().Get(context.Background(), csp.GetName(), metav1.GetOptions{})
+					if err != nil {
+						clusterWatcher.Log.Warnf("error getting csp", csp.GetName())
+						continue
+					}
+					if !reflect.DeepEqual(pol.Spec.Selector.MatchExpressions, common.RecommendedPolicies.MatchExpressions) {
+						pol.Spec.Selector.MatchExpressions = common.RecommendedPolicies.MatchExpressions
+						_, err := clusterWatcher.Secv1Client.SecurityV1().KubeArmorClusterPolicies().Update(context.Background(), pol, metav1.UpdateOptions{})
+						if err != nil {
+							clusterWatcher.Log.Warnf("error updating csp", csp.GetName())
+							continue
+						} else {
+							clusterWatcher.Log.Info("updated csp", csp.GetName())
+						}
+					}
+				} else {
+					clusterWatcher.Log.Info("created csp", csp.GetName())
+				}
+			}
+		}
+	case false:
+		policies, err := recommend.CRDFs.ReadDir(".")
+		if err != nil {
+			clusterWatcher.Log.Warnf("error reading policies FS", err)
+			return err
+		}
+		for _, policy := range policies {
+			yamlBytes, err := recommend.CRDFs.ReadFile(policy.Name())
+			if err != nil {
+				clusterWatcher.Log.Warnf("error reading csp", policy.Name())
+				continue
+			}
+			csp := &secv1.KubeArmorClusterPolicy{}
+			if err := runtime.DecodeInto(scheme.Codecs.UniversalDeserializer(), yamlBytes, csp); err != nil {
+				clusterWatcher.Log.Warnf("error decoding csp", policy.Name())
+				continue
+			}
+			if !policy.IsDir() {
+				err = clusterWatcher.Secv1Client.SecurityV1().KubeArmorClusterPolicies().Delete(context.Background(), csp.GetName(), metav1.DeleteOptions{})
+				if err != nil && !metav1errors.IsNotFound(err) {
+					clusterWatcher.Log.Warnf("error deleting csp", csp.GetName())
+					continue
+				} else {
+					clusterWatcher.Log.Info("deleted csp", csp.GetName())
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func UpdateRecommendedPolicyConfig(config *opv1.KubeArmorConfigSpec) bool {
+	updated := false
+	if config.RecommendedPolicies.Enable != common.RecommendedPolicies.Enable {
+		common.RecommendedPolicies.Enable = config.RecommendedPolicies.Enable
+		updated = true
+	}
+	if len(config.RecommendedPolicies.MatchExpressions) > 0 {
+		if reflect.DeepEqual(config.RecommendedPolicies.MatchExpressions, common.RecommendedPolicies.MatchExpressions) {
+			common.RecommendedPolicies.MatchExpressions = config.RecommendedPolicies.MatchExpressions
+			updated = true
+		}
+	}
+	return updated
+}
+
 func UpdateConfigMapData(config *opv1.KubeArmorConfigSpec) bool {
 	updated := false
 	if config.DefaultFilePosture != "" {

pkg/KubeArmorOperator/internal/controller/resources.go

@@ -748,6 +748,10 @@ func (clusterWatcher *ClusterWatcher) WatchRequiredResources() {
 			clusterWatcher.Log.Error(err.Error())
 		}
 
+		if err := clusterWatcher.WatchRecommendedPolicies(); err != nil {
+			installErr = err
+		}
+
 		// update operatingConfigCrd status to Running
 		if common.OperatorConfigCrd != nil {
 			if installErr != nil {