From 14a7062fb1de31332c27c4003cdaa931e9c231a5 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Wed, 28 Jun 2023 14:04:00 +0300 Subject: [PATCH 01/22] Change namespace to be a value on the Controller and StateApplier instead of coming from agentSpec. Modify objects to take the value from their own fields. --- .../state/components/cluster_configmap.go | 4 ++-- .../state/components/enforcer_deployment.go | 3 +-- .../state/components/enforcer_mutating_webhook.go | 10 +++++----- cbcontainers/state/components/enforcer_service.go | 2 +- .../state/components/enforcer_tls_secret.go | 2 +- .../image_scanning_reporter_deployment.go | 2 +- .../components/image_scanning_reporter_service.go | 2 +- .../state/components/monitor_deployment.go | 2 +- cbcontainers/state/components/registry_secret.go | 2 +- cbcontainers/state/components/resolver_service.go | 2 +- .../components/runtime_resolver_deployment.go | 2 +- .../state/components/sensor_daemon_set.go | 2 +- .../state/components/state_reporter_deployment.go | 2 +- cbcontainers/state/state_applier.go | 12 ++++-------- cbcontainers/state/state_applier_test.go | 3 +-- controllers/agent_defaults.go | 8 -------- controllers/cbcontainersagent_controller.go | 4 +++- controllers/cbcontainersagent_controller_test.go | 15 +++++++++------ 18 files changed, 35 insertions(+), 44 deletions(-) diff --git a/cbcontainers/state/components/cluster_configmap.go b/cbcontainers/state/components/cluster_configmap.go index 940b1001..be8bea39 100644 --- a/cbcontainers/state/components/cluster_configmap.go +++ b/cbcontainers/state/components/cluster_configmap.go @@ -35,12 +35,12 @@ func (obj *ConfigurationK8sObject) MutateK8sObject(k8sObject client.Object, agen return fmt.Errorf("expected ConfigMap K8s object") } - configMap.Namespace = agentSpec.Namespace + configMap.Namespace = obj.Namespace configMap.Data = map[string]string{ commonState.DataPlaneConfigmapAccountKey: agentSpec.Account, commonState.DataPlaneConfigmapClusterKey: agentSpec.ClusterName, commonState.DataPlaneConfigmapAgentVersionKey: agentSpec.Version, - commonState.DataPlaneConfigmapDataplaneNamespaceKey: agentSpec.Namespace, + commonState.DataPlaneConfigmapDataplaneNamespaceKey: configMap.Namespace, commonState.DataPlaneConfigmapApiSchemeKey: agentSpec.Gateways.ApiGateway.Scheme, commonState.DataPlaneConfigmapApiHostKey: agentSpec.Gateways.ApiGateway.Host, commonState.DataPlaneConfigmapApiPortKey: strconv.Itoa(agentSpec.Gateways.ApiGateway.Port), diff --git a/cbcontainers/state/components/enforcer_deployment.go b/cbcontainers/state/components/enforcer_deployment.go index 00803cde..f20c13df 100644 --- a/cbcontainers/state/components/enforcer_deployment.go +++ b/cbcontainers/state/components/enforcer_deployment.go @@ -84,8 +84,7 @@ func (obj *EnforcerDeploymentK8sObject) MutateK8sObject(k8sObject client.Object, if objectsDiffer(deployment.Spec.Template.Spec.ImagePullSecrets, desiredImagePullSecrets) { deployment.Spec.Template.Spec.ImagePullSecrets = desiredImagePullSecrets } - obj.Namespace = agentSpec.Namespace - deployment.Namespace = agentSpec.Namespace + deployment.Namespace = obj.Namespace obj.mutateAnnotations(deployment, enforcer) obj.mutateVolumes(&deployment.Spec.Template.Spec) obj.mutateAffinityAndNodeSelector(&deployment.Spec.Template.Spec, enforcer) diff --git a/cbcontainers/state/components/enforcer_mutating_webhook.go b/cbcontainers/state/components/enforcer_mutating_webhook.go index 6711eeaa..dca99af4 100644 --- a/cbcontainers/state/components/enforcer_mutating_webhook.go +++ b/cbcontainers/state/components/enforcer_mutating_webhook.go @@ -65,10 +65,10 @@ func (obj *EnforcerMutatingWebhookK8sObject) MutateK8sObject(k8sObject client.Ob enforcer := &agentSpec.Components.Basic.Enforcer obj.mutateWebhookConfigurationLabels(webhookConfiguration, enforcer) - return obj.mutateWebhooks(webhookConfiguration, enforcer, agentSpec.Namespace) + return obj.mutateWebhooks(webhookConfiguration, enforcer) } -func (obj *EnforcerMutatingWebhookK8sObject) mutateWebhooks(webhookConfiguration adapters.WebhookConfigurationAdapter, enforcer *cbcontainersv1.CBContainersEnforcerSpec, serviceNamespace string) error { +func (obj *EnforcerMutatingWebhookK8sObject) mutateWebhooks(webhookConfiguration adapters.WebhookConfigurationAdapter, enforcer *cbcontainersv1.CBContainersEnforcerSpec) error { var resourcesWebhookObj adapters.WebhookAdapter initializeWebhooks := false @@ -93,7 +93,7 @@ func (obj *EnforcerMutatingWebhookK8sObject) mutateWebhooks(webhookConfiguration resourcesWebhookObj = updatedWebhooks[0] } - obj.mutateResourcesWebhook(resourcesWebhookObj, enforcer.WebhookTimeoutSeconds, enforcer.FailurePolicy, serviceNamespace) + obj.mutateResourcesWebhook(resourcesWebhookObj, enforcer.WebhookTimeoutSeconds, enforcer.FailurePolicy) return nil } @@ -107,7 +107,7 @@ func (obj *EnforcerMutatingWebhookK8sObject) findWebhookByName(webhooks []adapte return nil, false } -func (obj *EnforcerMutatingWebhookK8sObject) mutateResourcesWebhook(resourcesWebhook adapters.WebhookAdapter, timeoutSeconds int32, failurePolicy, serviceNamespace string) { +func (obj *EnforcerMutatingWebhookK8sObject) mutateResourcesWebhook(resourcesWebhook adapters.WebhookAdapter, timeoutSeconds int32, failurePolicy string) { resourcesWebhook.SetName(MutatingWebhookName) resourcesWebhook.SetFailurePolicy(failurePolicy) resourcesWebhook.SetSideEffects(MutatingWebhookSideEffect) @@ -123,7 +123,7 @@ func (obj *EnforcerMutatingWebhookK8sObject) mutateResourcesWebhook(resourcesWeb } resourcesWebhook.SetCABundle(obj.tlsSecretValues.CaCert) resourcesWebhook.SetServiceName(EnforcerName) - resourcesWebhook.SetServiceNamespace(serviceNamespace) + resourcesWebhook.SetServiceNamespace(obj.ServiceNamespace) resourcesWebhook.SetServicePath(&MutatingWebhookPath) } diff --git a/cbcontainers/state/components/enforcer_service.go b/cbcontainers/state/components/enforcer_service.go index 336241f3..3cde3e13 100644 --- a/cbcontainers/state/components/enforcer_service.go +++ b/cbcontainers/state/components/enforcer_service.go @@ -45,7 +45,7 @@ func (obj *EnforcerServiceK8sObject) MutateK8sObject(k8sObject client.Object, ag service.Labels = enforcer.Labels service.Spec.Type = coreV1.ServiceTypeClusterIP - service.Namespace = agentSpec.Namespace + service.Namespace = obj.Namespace service.Spec.Selector = map[string]string{ EnforcerLabelKey: EnforcerName, } diff --git a/cbcontainers/state/components/enforcer_tls_secret.go b/cbcontainers/state/components/enforcer_tls_secret.go index 91ddb73f..f9b03cef 100644 --- a/cbcontainers/state/components/enforcer_tls_secret.go +++ b/cbcontainers/state/components/enforcer_tls_secret.go @@ -47,7 +47,7 @@ func (obj *EnforcerTlsK8sObject) MutateK8sObject(k8sObject client.Object, spec * return fmt.Errorf("expected Secret K8s object") } - secret.Namespace = spec.Namespace + secret.Namespace = obj.Namespace tlsSecretValues, err := obj.tlsSecretsValuesCreator.CreateTlsSecretsValues(types.NamespacedName{Name: EnforcerName, Namespace: obj.Namespace}) if err != nil { return err diff --git a/cbcontainers/state/components/image_scanning_reporter_deployment.go b/cbcontainers/state/components/image_scanning_reporter_deployment.go index f6c9f509..3696ba75 100644 --- a/cbcontainers/state/components/image_scanning_reporter_deployment.go +++ b/cbcontainers/state/components/image_scanning_reporter_deployment.go @@ -55,7 +55,7 @@ func (obj *ImageScanningReporterDeploymentK8sObject) MutateK8sObject(k8sObject c } clusterScanning := &agentSpec.Components.ClusterScanning - deployment.Namespace = agentSpec.Namespace + deployment.Namespace = obj.Namespace imageScanningReporter := &clusterScanning.ImageScanningReporter obj.initiateDeployment(deployment, agentSpec) obj.mutateLabels(deployment, imageScanningReporter) diff --git a/cbcontainers/state/components/image_scanning_reporter_service.go b/cbcontainers/state/components/image_scanning_reporter_service.go index 83e97926..68501329 100644 --- a/cbcontainers/state/components/image_scanning_reporter_service.go +++ b/cbcontainers/state/components/image_scanning_reporter_service.go @@ -43,7 +43,7 @@ func (obj *ImageScanningReporterServiceK8sObject) MutateK8sObject(k8sObject clie imageScanningReporter := &agentSpec.Components.ClusterScanning.ImageScanningReporter - service.Namespace = agentSpec.Namespace + service.Namespace = obj.Namespace service.Labels = imageScanningReporter.Labels service.Spec.Type = coreV1.ServiceTypeClusterIP service.Spec.Selector = map[string]string{ diff --git a/cbcontainers/state/components/monitor_deployment.go b/cbcontainers/state/components/monitor_deployment.go index c887def4..4a757b48 100644 --- a/cbcontainers/state/components/monitor_deployment.go +++ b/cbcontainers/state/components/monitor_deployment.go @@ -76,7 +76,7 @@ func (obj *MonitorDeploymentK8sObject) MutateK8sObject(k8sObject client.Object, deployment.Spec.Template.ObjectMeta.Annotations = make(map[string]string) } - deployment.Namespace = agentSpec.Namespace + deployment.Namespace = obj.Namespace deployment.Spec.Replicas = &MonitorReplicas deployment.ObjectMeta.Labels = desiredLabels deployment.Spec.Selector.MatchLabels = desiredLabels diff --git a/cbcontainers/state/components/registry_secret.go b/cbcontainers/state/components/registry_secret.go index a8083643..2fd14250 100644 --- a/cbcontainers/state/components/registry_secret.go +++ b/cbcontainers/state/components/registry_secret.go @@ -46,7 +46,7 @@ func (obj *RegistrySecretK8sObject) MutateK8sObject(k8sObject client.Object, spe secret.Type = obj.registrySecretValues.Type secret.Data = obj.registrySecretValues.Data - secret.Namespace = spec.Namespace + secret.Namespace = obj.Namespace return nil } diff --git a/cbcontainers/state/components/resolver_service.go b/cbcontainers/state/components/resolver_service.go index 94b8ea17..9a321ad1 100644 --- a/cbcontainers/state/components/resolver_service.go +++ b/cbcontainers/state/components/resolver_service.go @@ -46,7 +46,7 @@ func (obj *ResolverServiceK8sObject) MutateK8sObject(k8sObject client.Object, ag service.Spec.Type = coreV1.ServiceTypeClusterIP service.Spec.ClusterIP = coreV1.ClusterIPNone - service.Namespace = agentSpec.Namespace + service.Namespace = obj.Namespace service.Spec.Selector = map[string]string{ resolverLabelKey: ResolverName, } diff --git a/cbcontainers/state/components/runtime_resolver_deployment.go b/cbcontainers/state/components/runtime_resolver_deployment.go index 3f9dd247..53354ee7 100644 --- a/cbcontainers/state/components/runtime_resolver_deployment.go +++ b/cbcontainers/state/components/runtime_resolver_deployment.go @@ -89,7 +89,7 @@ func (obj *ResolverDeploymentK8sObject) MutateK8sObject(k8sObject client.Object, } } - deployment.Namespace = agentSpec.Namespace + deployment.Namespace = obj.Namespace deployment.Spec.Replicas = replicasCount deployment.ObjectMeta.Labels = desiredLabels deployment.Spec.Selector.MatchLabels = desiredLabels diff --git a/cbcontainers/state/components/sensor_daemon_set.go b/cbcontainers/state/components/sensor_daemon_set.go index f790783c..13a16f1d 100644 --- a/cbcontainers/state/components/sensor_daemon_set.go +++ b/cbcontainers/state/components/sensor_daemon_set.go @@ -120,7 +120,7 @@ func (obj *SensorDaemonSetK8sObject) MutateK8sObject(k8sObject client.Object, ag daemonSet.Spec.Template.Spec.HostPID = false } - daemonSet.Namespace = agentSpec.Namespace + daemonSet.Namespace = obj.Namespace obj.mutateLabels(daemonSet, agentSpec) obj.mutateAnnotations(daemonSet, agentSpec) obj.mutateVolumes(daemonSet, agentSpec) diff --git a/cbcontainers/state/components/state_reporter_deployment.go b/cbcontainers/state/components/state_reporter_deployment.go index d1d00d35..e49fbe27 100644 --- a/cbcontainers/state/components/state_reporter_deployment.go +++ b/cbcontainers/state/components/state_reporter_deployment.go @@ -72,7 +72,7 @@ func (obj *StateReporterDeploymentK8sObject) MutateK8sObject(k8sObject client.Ob deployment.Spec.Template.ObjectMeta.Annotations = make(map[string]string) } - deployment.Namespace = agentSpec.Namespace + deployment.Namespace = obj.Namespace deployment.Spec.Replicas = &StateReporterReplicas deployment.ObjectMeta.Labels = desiredLabels deployment.Spec.Selector.MatchLabels = desiredLabels diff --git a/cbcontainers/state/state_applier.go b/cbcontainers/state/state_applier.go index 6cbc6741..6c597d50 100644 --- a/cbcontainers/state/state_applier.go +++ b/cbcontainers/state/state_applier.go @@ -39,9 +39,10 @@ type StateApplier struct { imageScanningReporterService *components.ImageScanningReporterServiceK8sObject applier AgentComponentApplier log logr.Logger + agentNamespace string } -func NewStateApplier(apiReader client.Reader, agentComponentApplier AgentComponentApplier, k8sVersion string, tlsSecretsValuesCreator components.TlsSecretsValuesCreator, log logr.Logger) *StateApplier { +func NewStateApplier(apiReader client.Reader, agentComponentApplier AgentComponentApplier, k8sVersion, agentNamespace string, tlsSecretsValuesCreator components.TlsSecretsValuesCreator, log logr.Logger) *StateApplier { return &StateApplier{ desiredConfigMap: components.NewConfigurationK8sObject(), desiredRegistrySecret: components.NewRegistrySecretK8sObject(), @@ -60,6 +61,7 @@ func NewStateApplier(apiReader client.Reader, agentComponentApplier AgentCompone imageScanningReporterService: components.NewImageScanningReporterServiceK8sObject(), applier: agentComponentApplier, log: log, + agentNamespace: agentNamespace, } } @@ -69,13 +71,7 @@ func (c *StateApplier) GetPriorityClassEmptyK8sObject() client.Object { func (c *StateApplier) ApplyDesiredState(ctx context.Context, agentSpec *cbcontainersv1.CBContainersAgentSpec, registrySecret *models.RegistrySecretValues, setOwner applymentOptions.OwnerSetter) (bool, error) { applyOptions := applymentOptions.NewApplyOptions().SetOwnerSetter(setOwner) - - // The namespace field of the agent spec should always be populated, because it has a default value - // but just in case include this check here in case it turns out to be empty in the future. - // By default all objects have the "cbcontainers-dataplane" as namespace. - if agentSpec.Namespace != "" { - c.setNamespace(agentSpec.Namespace) - } + c.setNamespace(c.agentNamespace) coreMutated, err := c.applyCoreComponents(ctx, agentSpec, registrySecret, applyOptions) if err != nil { diff --git a/cbcontainers/state/state_applier_test.go b/cbcontainers/state/state_applier_test.go index 5ee90bb2..4a4b6140 100644 --- a/cbcontainers/state/state_applier_test.go +++ b/cbcontainers/state/state_applier_test.go @@ -148,7 +148,6 @@ func testStateApplier(t *testing.T, setup StateApplierTestSetup, k8sVersion, nam agentSpec := &cbcontainersv1.CBContainersAgentSpec{ Account: Account, ClusterName: Cluster, - Namespace: namespace, Gateways: cbcontainersv1.CBContainersGatewaysSpec{ ApiGateway: cbcontainersv1.CBContainersApiGatewaySpec{ Scheme: ApiGateWayScheme, @@ -190,7 +189,7 @@ func testStateApplier(t *testing.T, setup StateApplierTestSetup, k8sVersion, nam setup(mockObjects) - stateApplier := state.NewStateApplier(testUtilsMocks.NewMockReader(ctrl), mockObjects.componentApplier, k8sVersion, mockObjects.secretValuesCreator, logrTesting.NewTestLogger(t)) + stateApplier := state.NewStateApplier(testUtilsMocks.NewMockReader(ctrl), mockObjects.componentApplier, k8sVersion, namespace, mockObjects.secretValuesCreator, logrTesting.NewTestLogger(t)) return stateApplier.ApplyDesiredState(context.Background(), agentSpec, &models.RegistrySecretValues{}, nil) } diff --git a/controllers/agent_defaults.go b/controllers/agent_defaults.go index 04cc4424..39faca9b 100644 --- a/controllers/agent_defaults.go +++ b/controllers/agent_defaults.go @@ -2,7 +2,6 @@ package controllers import ( cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" - "github.com/vmware/cbcontainers-operator/cbcontainers/state/common" ) func (r *CBContainersAgentController) setAgentDefaults(agentSpec *cbcontainersv1.CBContainersAgentSpec) error { @@ -16,13 +15,6 @@ func (r *CBContainersAgentController) setAgentDefaults(agentSpec *cbcontainersv1 return err } - // The namespace field of the agent spec should always be populated, because it has a default value, - // but just in case include this check here in case it turns out to be empty in the future. - // By default all objects have the "cbcontainers-dataplane" as namespace. - if agentSpec.Namespace == "" { - agentSpec.Namespace = common.DataPlaneNamespaceName - } - return nil } diff --git a/controllers/cbcontainersagent_controller.go b/controllers/cbcontainersagent_controller.go index b367844a..3f7c4dbd 100644 --- a/controllers/cbcontainersagent_controller.go +++ b/controllers/cbcontainersagent_controller.go @@ -52,6 +52,8 @@ type CBContainersAgentController struct { ClusterProcessor AgentProcessor StateApplier StateApplier K8sVersion string + // Namespace is the kubernetes namespace for all agent components + Namespace string } func (r *CBContainersAgentController) getContainersAgentObject(ctx context.Context) (*cbcontainersv1.CBContainersAgent, error) { @@ -137,7 +139,7 @@ func (r *CBContainersAgentController) getRegistrySecretValues(ctx context.Contex } func (r *CBContainersAgentController) getAccessToken(ctx context.Context, cbContainersCluster *cbcontainersv1.CBContainersAgent) (string, error) { - accessTokenSecretNamespacedName := types.NamespacedName{Name: cbContainersCluster.Spec.AccessTokenSecretName, Namespace: cbContainersCluster.Spec.Namespace} + accessTokenSecretNamespacedName := types.NamespacedName{Name: cbContainersCluster.Spec.AccessTokenSecretName, Namespace: r.Namespace} accessTokenSecret := &corev1.Secret{} if err := r.Get(ctx, accessTokenSecretNamespacedName, accessTokenSecret); err != nil { return "", fmt.Errorf("couldn't find access token secret k8s object: %v", err) diff --git a/controllers/cbcontainersagent_controller_test.go b/controllers/cbcontainersagent_controller_test.go index e86836f2..7ab2b3af 100644 --- a/controllers/cbcontainersagent_controller_test.go +++ b/controllers/cbcontainersagent_controller_test.go @@ -32,6 +32,8 @@ type ClusterControllerTestMocks struct { const ( MyClusterTokenValue = "my-token-value" + // agentNamespace helps validate we don't depend on a hardcoded namespace anywhere + agentNamespace = "dummy-namespace" ) var ( @@ -70,9 +72,10 @@ func testCBContainersClusterController(t *testing.T, setups ...SetupClusterContr } controller := &controllers.CBContainersAgentController{ - Client: mocksObjects.client, - Log: logrTesting.NewTestLogger(t), - Scheme: &runtime.Scheme{}, + Client: mocksObjects.client, + Log: logrTesting.NewTestLogger(t), + Scheme: &runtime.Scheme{}, + Namespace: agentNamespace, ClusterProcessor: mocksObjects.ClusterProcessor, StateApplier: mocksObjects.StateApplier, @@ -90,7 +93,7 @@ func setupClusterCustomResource(testMocks *ClusterControllerTestMocks) { } func setUpTokenSecretValues(testMocks *ClusterControllerTestMocks) { - accessTokenSecretNamespacedName := types.NamespacedName{Name: ClusterAccessTokenSecretName, Namespace: commonState.DataPlaneNamespaceName} + accessTokenSecretNamespacedName := types.NamespacedName{Name: ClusterAccessTokenSecretName, Namespace: agentNamespace} testMocks.client.EXPECT().Get(testMocks.ctx, accessTokenSecretNamespacedName, &corev1.Secret{}). Do(func(ctx context.Context, namespacedName types.NamespacedName, secret *corev1.Secret, _ ...interface{}) { secret.Data = map[string][]byte{ @@ -132,7 +135,7 @@ func TestFindingMoreThanOneClusterResourceShouldReturnError(t *testing.T) { func TestGetTokenSecretErrorShouldReturnError(t *testing.T) { _, err := testCBContainersClusterController(t, setupClusterCustomResource, func(testMocks *ClusterControllerTestMocks) { - accessTokenSecretNamespacedName := types.NamespacedName{Name: ClusterAccessTokenSecretName, Namespace: commonState.DataPlaneNamespaceName} + accessTokenSecretNamespacedName := types.NamespacedName{Name: ClusterAccessTokenSecretName, Namespace: agentNamespace} testMocks.client.EXPECT().Get(testMocks.ctx, accessTokenSecretNamespacedName, &corev1.Secret{}).Return(fmt.Errorf("")) }) @@ -141,7 +144,7 @@ func TestGetTokenSecretErrorShouldReturnError(t *testing.T) { func TestTokenSecretWithoutTokenValueShouldReturnError(t *testing.T) { _, err := testCBContainersClusterController(t, setupClusterCustomResource, func(testMocks *ClusterControllerTestMocks) { - accessTokenSecretNamespacedName := types.NamespacedName{Name: ClusterAccessTokenSecretName, Namespace: commonState.DataPlaneNamespaceName} + accessTokenSecretNamespacedName := types.NamespacedName{Name: ClusterAccessTokenSecretName, Namespace: agentNamespace} testMocks.client.EXPECT().Get(testMocks.ctx, accessTokenSecretNamespacedName, &corev1.Secret{}).Return(nil) }) From f83633eff131be044875527910e0d1e467cbb87a Mon Sep 17 00:00:00 2001 From: ltsonov Date: Wed, 28 Jun 2023 14:20:42 +0300 Subject: [PATCH 02/22] Add env var to pass the operator's namespace into the code and use it for the agent --- config/manager/manager.yaml | 7 ++++- main.go | 59 ++++++++++++++++++++++++------------- 2 files changed, 45 insertions(+), 21 deletions(-) diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 472e6f2b..8a3b8821 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -75,5 +75,10 @@ spec: requests: cpu: 100m memory: 64Mi + env: + - name: OPERATOR_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace serviceAccountName: operator - terminationGracePeriodSeconds: 10 + terminationGracePeriodSeconds: 10 \ No newline at end of file diff --git a/main.go b/main.go index 4c44a08c..54a69295 100644 --- a/main.go +++ b/main.go @@ -23,10 +23,13 @@ import ( "github.com/vmware/cbcontainers-operator/cbcontainers/state" "github.com/vmware/cbcontainers-operator/cbcontainers/state/agent_applyment" "github.com/vmware/cbcontainers-operator/cbcontainers/state/applyment" + "github.com/vmware/cbcontainers-operator/cbcontainers/state/common" "github.com/vmware/cbcontainers-operator/cbcontainers/state/operator" + "go.uber.org/zap/zapcore" "os" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" coreV1 "k8s.io/api/core/v1" @@ -72,6 +75,7 @@ func main() { "Enabling this will ensure there is only one active controller manager.") opts := zap.Options{ Development: true, + TimeEncoder: zapcore.RFC3339TimeEncoder, } opts.BindFlags(flag.CommandLine) flag.Parse() @@ -92,25 +96,7 @@ func main() { os.Exit(1) } - setupLog.Info(fmt.Sprintf("Getting Cluster Identifier: %v uid", NamespaceIdentifier)) - namespace := &coreV1.Namespace{} - apiReader := mgr.GetAPIReader() - if err = apiReader.Get(context.Background(), client.ObjectKey{Namespace: NamespaceIdentifier, Name: NamespaceIdentifier}, namespace); err != nil { - setupLog.Error(err, fmt.Sprintf("unable to get the %v namespace", NamespaceIdentifier)) - os.Exit(1) - } - clusterIdentifier := string(namespace.UID) - - setupLog.Info(fmt.Sprintf("Cluster Identifier: %v", clusterIdentifier)) - - setupLog.Info("Getting Nodes list") - nodesList := &coreV1.NodeList{} - if err := apiReader.List(context.Background(), nodesList); err != nil || nodesList.Items == nil || len(nodesList.Items) < 1 { - setupLog.Error(err, "couldn't get nodes list") - os.Exit(1) - } - k8sVersion := nodesList.Items[0].Status.NodeInfo.KubeletVersion - setupLog.Info(fmt.Sprintf("K8s version is: %v", k8sVersion)) + clusterIdentifier, k8sVersion, operatorNamespace := extractConfigurationVariables(mgr) cbContainersAgentLogger := ctrl.Log.WithName("controllers").WithName("CBContainersAgent") if err = (&controllers.CBContainersAgentController{ @@ -118,8 +104,9 @@ func main() { Log: cbContainersAgentLogger, Scheme: mgr.GetScheme(), K8sVersion: k8sVersion, + Namespace: operatorNamespace, ClusterProcessor: processors.NewAgentProcessor(cbContainersAgentLogger, processors.NewDefaultGatewayCreator(), operator.NewEnvVersionProvider(), clusterIdentifier), - StateApplier: state.NewStateApplier(mgr.GetAPIReader(), agent_applyment.NewAgentComponent(applyment.NewComponentApplier(mgr.GetClient())), k8sVersion, certificatesUtils.NewCertificateCreator(), cbContainersAgentLogger), + StateApplier: state.NewStateApplier(mgr.GetAPIReader(), agent_applyment.NewAgentComponent(applyment.NewComponentApplier(mgr.GetClient())), k8sVersion, operatorNamespace, certificatesUtils.NewCertificateCreator(), cbContainersAgentLogger), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "CBContainersAgent") os.Exit(1) @@ -142,3 +129,35 @@ func main() { os.Exit(1) } } + +func extractConfigurationVariables(mgr manager.Manager) (clusterIdentifier string, k8sVersion string, operatorNamespace string) { + setupLog.Info(fmt.Sprintf("Getting Cluster Identifier: %v uid", NamespaceIdentifier)) + namespace := &coreV1.Namespace{} + apiReader := mgr.GetAPIReader() + if err := apiReader.Get(context.Background(), client.ObjectKey{Namespace: NamespaceIdentifier, Name: NamespaceIdentifier}, namespace); err != nil { + setupLog.Error(err, fmt.Sprintf("unable to get the %v namespace", NamespaceIdentifier)) + os.Exit(1) + } + clusterIdentifier = string(namespace.UID) + + setupLog.Info(fmt.Sprintf("Cluster Identifier: %v", clusterIdentifier)) + + setupLog.Info("Getting Nodes list") + nodesList := &coreV1.NodeList{} + if err := apiReader.List(context.Background(), nodesList); err != nil || nodesList.Items == nil || len(nodesList.Items) < 1 { + setupLog.Error(err, "couldn't get nodes list") + os.Exit(1) + } + k8sVersion = nodesList.Items[0].Status.NodeInfo.KubeletVersion + setupLog.Info(fmt.Sprintf("K8s version is: %v", k8sVersion)) + + setupLog.Info("Getting the namespace where operator is running and which should host the agent") + operatorNamespace = os.Getenv("OPERATOR_NAMESPACE") + if operatorNamespace == "" { + setupLog.Info(fmt.Sprintf("Operator namespace variable was not found. Falling back to default %s", common.DataPlaneNamespaceName)) + operatorNamespace = common.DataPlaneNamespaceName + } + setupLog.Info(fmt.Sprintf("Operator and agent namespace: %s", operatorNamespace)) + + return +} From 2992c16419f91c9b63ff3f38d30b944df262a4e7 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Wed, 28 Jun 2023 15:55:58 +0300 Subject: [PATCH 03/22] Add a description to the Namespace field to indicate it is deprecated and will not have any effect. --- api/v1/cbcontainersagent_types.go | 5 +++++ ...perator.containers.carbonblack.io_cbcontainersagents.yaml | 2 ++ ...perator.containers.carbonblack.io_cbcontainersagents.yaml | 2 ++ 3 files changed, 9 insertions(+) diff --git a/api/v1/cbcontainersagent_types.go b/api/v1/cbcontainersagent_types.go index 84263102..4dca372e 100644 --- a/api/v1/cbcontainersagent_types.go +++ b/api/v1/cbcontainersagent_types.go @@ -30,7 +30,12 @@ type CBContainersAgentSpec struct { ClusterName string `json:"clusterName,required"` Version string `json:"version,required"` Gateways CBContainersGatewaysSpec `json:"gateways,required"` + // The field below remains to avoid moving the CRD from v1 to v2. + // It MUST not be used as agent namespace should be controlled outside the operator itself. + // This is because a custom namespace in the CRD requires high privileges by the operator across the whole cluster to be able to "switch" namespaces on demand. + // +kubebuilder:default:="cbcontainers-dataplane" + // Namespace is deprecated and the value has no effect. Do not use. Namespace string `json:"namespace,omitempty"` // +kubebuilder:default:="cbcontainers-access-token" AccessTokenSecretName string `json:"accessTokenSecretName,omitempty"` diff --git a/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml b/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml index 1009057e..ed0b9ed6 100644 --- a/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml +++ b/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml @@ -6036,6 +6036,8 @@ spec: type: object namespace: default: cbcontainers-dataplane + description: Namespace is deprecated and the value has no effect. + Do not use. type: string version: type: string diff --git a/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml b/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml index 233baf9c..f7eb8a3c 100644 --- a/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml +++ b/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml @@ -5608,6 +5608,8 @@ spec: - runtimeEventsGateway type: object namespace: + description: Namespace is deprecated and the value has no effect. Do + not use. type: string version: type: string From e7501ea2fd9cf98e8c4d63e2d809350af6cd8c29 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Wed, 28 Jun 2023 16:56:12 +0300 Subject: [PATCH 04/22] Reduce operator workload permissions to a role instead of ClusterRole --- config/rbac/role.yaml | 68 ++++++++++++--------- config/rbac/role_binding.yaml | 16 ++++- controllers/cbcontainersagent_controller.go | 8 ++- main.go | 21 ++++--- 4 files changed, 72 insertions(+), 41 deletions(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 6e6f84c5..4c8e1bdd 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -13,34 +13,6 @@ rules: - validatingwebhookconfigurations verbs: - '*' -- apiGroups: - - apps - - "" - resources: - - daemonsets - - deployments - - services - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - configmaps - - secrets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - apiGroups: - "" resources: @@ -93,3 +65,43 @@ rules: - priorityclasses verbs: - '*' + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + creationTimestamp: null + name: manager-role + namespace: cbcontainers-dataplane +rules: +- apiGroups: + - apps + - "" + resources: + - daemonsets + - deployments + - services + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch + diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml index cb9f682d..7cd6f8cd 100644 --- a/config/rbac/role_binding.yaml +++ b/config/rbac/role_binding.yaml @@ -1,7 +1,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: manager-rolebinding + name: manager-clusterrolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -12,6 +12,20 @@ subjects: namespace: system --- apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: manager-rolebinding + namespace: system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: manager-role +subjects: + - kind: ServiceAccount + name: operator + namespace: system +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: state-reporter-rolebinding diff --git a/controllers/cbcontainersagent_controller.go b/controllers/cbcontainersagent_controller.go index 3f7c4dbd..152299e7 100644 --- a/controllers/cbcontainersagent_controller.go +++ b/controllers/cbcontainersagent_controller.go @@ -73,16 +73,19 @@ func (r *CBContainersAgentController) getContainersAgentObject(ctx context.Conte return &cbContainersAgentsList.Items[0], nil } +// The following values must be kept in-sync with constants for generated RBAC to work properly: +// - default dataplane namespace (see common.DataPlaneNamespaceName) + // +kubebuilder:rbac:groups=operator.containers.carbonblack.io,resources=cbcontainersagents,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=operator.containers.carbonblack.io,resources=cbcontainersagents/status,verbs=get;update;patch // +kubebuilder:rbac:groups=operator.containers.carbonblack.io,resources=cbcontainersagents/finalizers,verbs=update -// +kubebuilder:rbac:groups=core,resources={configmaps,secrets},verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=scheduling.k8s.io,resources=priorityclasses,verbs=* -// +kubebuilder:rbac:groups={apps,core},resources={deployments,services,daemonsets},verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=admissionregistration.k8s.io,resources={validatingwebhookconfigurations,mutatingwebhookconfigurations},verbs=* // +kubebuilder:rbac:groups={core},resources={nodes},verbs=list // +kubebuilder:rbac:groups={core},resources={namespaces},verbs=get // +kubebuilder:rbac:groups={policy},resources={podsecuritypolicies},verbs=use,resourceNames={cbcontainers-manager-psp} +// +kubebuilder:rbac:groups={apps,core},resources={deployments,services,daemonsets},namespace=cbcontainers-dataplane,verbs=get;list;watch;create;update;patch;delete;deletecollection +// +kubebuilder:rbac:groups=core,resources={configmaps,secrets},namespace=cbcontainers-dataplane,verbs=get;list;watch;create;update;patch;delete;deletecollection func (r *CBContainersAgentController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { r.Log.Info("\n\n") @@ -96,6 +99,7 @@ func (r *CBContainersAgentController) Reconcile(ctx context.Context, req ctrl.Re } if cbContainersAgent == nil { + r.Log.Info("No CBContainersAgent object found") return ctrl.Result{}, nil } diff --git a/main.go b/main.go index 54a69295..b5a727ca 100644 --- a/main.go +++ b/main.go @@ -82,6 +82,14 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + setupLog.Info("Getting the namespace where operator is running and which should host the agent") + operatorNamespace := os.Getenv("OPERATOR_NAMESPACE") + if operatorNamespace == "" { + setupLog.Info(fmt.Sprintf("Operator namespace variable was not found. Falling back to default %s", common.DataPlaneNamespaceName)) + operatorNamespace = common.DataPlaneNamespaceName + } + setupLog.Info(fmt.Sprintf("Operator and agent namespace: %s", operatorNamespace)) + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, MetricsBindAddress: metricsAddr, @@ -90,13 +98,14 @@ func main() { LeaderElection: enableLeaderElection, LeaderElectionID: "d27fd235.operator.containers.carbonblack.io", Logger: ctrl.Log, + Namespace: operatorNamespace, }) if err != nil { setupLog.Error(err, "unable to start manager") os.Exit(1) } - clusterIdentifier, k8sVersion, operatorNamespace := extractConfigurationVariables(mgr) + clusterIdentifier, k8sVersion := extractConfigurationVariables(mgr) cbContainersAgentLogger := ctrl.Log.WithName("controllers").WithName("CBContainersAgent") if err = (&controllers.CBContainersAgentController{ @@ -130,7 +139,7 @@ func main() { } } -func extractConfigurationVariables(mgr manager.Manager) (clusterIdentifier string, k8sVersion string, operatorNamespace string) { +func extractConfigurationVariables(mgr manager.Manager) (clusterIdentifier string, k8sVersion string) { setupLog.Info(fmt.Sprintf("Getting Cluster Identifier: %v uid", NamespaceIdentifier)) namespace := &coreV1.Namespace{} apiReader := mgr.GetAPIReader() @@ -151,13 +160,5 @@ func extractConfigurationVariables(mgr manager.Manager) (clusterIdentifier strin k8sVersion = nodesList.Items[0].Status.NodeInfo.KubeletVersion setupLog.Info(fmt.Sprintf("K8s version is: %v", k8sVersion)) - setupLog.Info("Getting the namespace where operator is running and which should host the agent") - operatorNamespace = os.Getenv("OPERATOR_NAMESPACE") - if operatorNamespace == "" { - setupLog.Info(fmt.Sprintf("Operator namespace variable was not found. Falling back to default %s", common.DataPlaneNamespaceName)) - operatorNamespace = common.DataPlaneNamespaceName - } - setupLog.Info(fmt.Sprintf("Operator and agent namespace: %s", operatorNamespace)) - return } From 81d45b6aba5fd7e55eeccf3c5d39efc6ec9837f7 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Fri, 30 Jun 2023 13:28:37 +0300 Subject: [PATCH 05/22] Move dataplane RBAC to a subfolder to separate operator vs dataplane --- .../rbac/{ => dataplane}/dataplane_roles.yaml | 0 config/rbac/dataplane/role_binding.yaml | 91 +++++++++++++++++++ config/rbac/dataplane/service_account.yaml | 48 ++++++++++ 3 files changed, 139 insertions(+) rename config/rbac/{ => dataplane}/dataplane_roles.yaml (100%) create mode 100644 config/rbac/dataplane/role_binding.yaml create mode 100644 config/rbac/dataplane/service_account.yaml diff --git a/config/rbac/dataplane_roles.yaml b/config/rbac/dataplane/dataplane_roles.yaml similarity index 100% rename from config/rbac/dataplane_roles.yaml rename to config/rbac/dataplane/dataplane_roles.yaml diff --git a/config/rbac/dataplane/role_binding.yaml b/config/rbac/dataplane/role_binding.yaml new file mode 100644 index 00000000..e54e864e --- /dev/null +++ b/config/rbac/dataplane/role_binding.yaml @@ -0,0 +1,91 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: state-reporter-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: state-reporter-role +subjects: + - kind: ServiceAccount + name: state-reporter + namespace: system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: enforcer-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: enforcer-role +subjects: + - kind: ServiceAccount + name: enforcer + namespace: system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: monitor-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: monitor-role +subjects: + - kind: ServiceAccount + name: monitor + namespace: system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: image-scanning-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: image-scanning-role +subjects: + - kind: ServiceAccount + name: image-scanning + namespace: system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: runtime-resolver-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: runtime-resolver-role +subjects: + - kind: ServiceAccount + name: runtime-resolver + namespace: system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: agent-node-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: agent-node-role +subjects: + - kind: ServiceAccount + name: agent-node + namespace: system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: runtime-resolver-to-agent-node-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: runtime-resolver-role +subjects: + - kind: ServiceAccount + name: agent-node + namespace: system diff --git a/config/rbac/dataplane/service_account.yaml b/config/rbac/dataplane/service_account.yaml new file mode 100644 index 00000000..926160b3 --- /dev/null +++ b/config/rbac/dataplane/service_account.yaml @@ -0,0 +1,48 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: state-reporter + namespace: system +imagePullSecrets: + - name: cbcontainers-operator-public-registry-secret +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: enforcer + namespace: system +imagePullSecrets: + - name: cbcontainers-operator-public-registry-secret +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: monitor + namespace: system +imagePullSecrets: + - name: cbcontainers-operator-public-registry-secret +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: image-scanning + namespace: system +imagePullSecrets: + - name: cbcontainers-operator-public-registry-secret +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: runtime-resolver + namespace: system +imagePullSecrets: + - name: cbcontainers-operator-public-registry-secret +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: agent-node + namespace: system +imagePullSecrets: + - name: cbcontainers-operator-public-registry-secret + From b2081f6b5cd86b345a023d1bfb4f262ff80ec917 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Fri, 30 Jun 2023 13:29:33 +0300 Subject: [PATCH 06/22] Commit generated file diff --- config/rbac/role.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 4c8e1bdd..adecf5bc 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -104,4 +104,3 @@ rules: - patch - update - watch - From 5b1187a2639fd075e6f511ec33690ea2e09b19c9 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Fri, 30 Jun 2023 13:29:48 +0300 Subject: [PATCH 07/22] Add dataplane files to the kustomization --- config/rbac/kustomization.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 40dca000..93b3bacb 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -1,6 +1,8 @@ resources: - role.yaml -- dataplane_roles.yaml +- dataplane/dataplane_roles.yaml +- dataplane/role_binding.yaml +- dataplane/service_account.yaml - role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml From 35235830dba994a1c15c8f83a8eaabeb578f9159 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Fri, 30 Jun 2023 13:41:04 +0300 Subject: [PATCH 08/22] Remove duplicated Dataplane items --- config/rbac/role_binding.yaml | 91 -------------------------------- config/rbac/service_account.yaml | 48 ----------------- 2 files changed, 139 deletions(-) diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml index 7cd6f8cd..72ca072a 100644 --- a/config/rbac/role_binding.yaml +++ b/config/rbac/role_binding.yaml @@ -24,94 +24,3 @@ subjects: - kind: ServiceAccount name: operator namespace: system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: state-reporter-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: state-reporter-role -subjects: - - kind: ServiceAccount - name: state-reporter - namespace: system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: enforcer-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: enforcer-role -subjects: - - kind: ServiceAccount - name: enforcer - namespace: system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: monitor-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: monitor-role -subjects: - - kind: ServiceAccount - name: monitor - namespace: system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: image-scanning-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: image-scanning-role -subjects: - - kind: ServiceAccount - name: image-scanning - namespace: system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: runtime-resolver-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: runtime-resolver-role -subjects: - - kind: ServiceAccount - name: runtime-resolver - namespace: system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: agent-node-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: agent-node-role -subjects: - - kind: ServiceAccount - name: agent-node - namespace: system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: runtime-resolver-to-agent-node-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: runtime-resolver-role -subjects: - - kind: ServiceAccount - name: agent-node - namespace: system diff --git a/config/rbac/service_account.yaml b/config/rbac/service_account.yaml index e209e470..a63e646e 100644 --- a/config/rbac/service_account.yaml +++ b/config/rbac/service_account.yaml @@ -14,52 +14,4 @@ metadata: namespace: system imagePullSecrets: - name: cbcontainers-operator-public-registry-secret ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: state-reporter - namespace: system -imagePullSecrets: - - name: cbcontainers-operator-public-registry-secret ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: enforcer - namespace: system -imagePullSecrets: - - name: cbcontainers-operator-public-registry-secret ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: monitor - namespace: system -imagePullSecrets: - - name: cbcontainers-operator-public-registry-secret ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: image-scanning - namespace: system -imagePullSecrets: - - name: cbcontainers-operator-public-registry-secret ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: runtime-resolver - namespace: system -imagePullSecrets: - - name: cbcontainers-operator-public-registry-secret ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: agent-node - namespace: system -imagePullSecrets: - - name: cbcontainers-operator-public-registry-secret From 4dce536b60ddc7f893db3f866f099a672d6bdd92 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Fri, 30 Jun 2023 13:41:51 +0300 Subject: [PATCH 09/22] Keep the old ClusterRoleBinding name to avoid duplicates if redeploying a newer operator. Rename the local RoleBinding instead to avoid conflicts. --- config/rbac/role_binding.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml index 72ca072a..dff03bf6 100644 --- a/config/rbac/role_binding.yaml +++ b/config/rbac/role_binding.yaml @@ -1,7 +1,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: manager-clusterrolebinding + name: manager-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -14,7 +14,7 @@ subjects: apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: manager-rolebinding + name: manager-namespaced-rolebinding namespace: system roleRef: apiGroup: rbac.authorization.k8s.io From da611d4fd3b749603e7334a9e06637dcbfacdabf Mon Sep 17 00:00:00 2001 From: ltsonov Date: Fri, 30 Jun 2023 16:38:32 +0300 Subject: [PATCH 10/22] Restrict PriorityClass RBAC --- config/rbac/role.yaml | 15 ++++++++++++++- controllers/cbcontainersagent_controller.go | 4 +++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index adecf5bc..50adeb80 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -64,7 +64,20 @@ rules: resources: - priorityclasses verbs: - - '*' + - create + - list + - watch +- apiGroups: + - scheduling.k8s.io + resourceNames: + - cbcontainers-dataplane-priority-class + resources: + - priorityclasses + verbs: + - delete + - get + - patch + - update --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/controllers/cbcontainersagent_controller.go b/controllers/cbcontainersagent_controller.go index 152299e7..4f5fb2fc 100644 --- a/controllers/cbcontainersagent_controller.go +++ b/controllers/cbcontainersagent_controller.go @@ -75,11 +75,13 @@ func (r *CBContainersAgentController) getContainersAgentObject(ctx context.Conte // The following values must be kept in-sync with constants for generated RBAC to work properly: // - default dataplane namespace (see common.DataPlaneNamespaceName) +// - cluster-wide dataplane priority class (see common.DataPlanePriorityClassName) - avoids access to all priority classes on the cluster // +kubebuilder:rbac:groups=operator.containers.carbonblack.io,resources=cbcontainersagents,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=operator.containers.carbonblack.io,resources=cbcontainersagents/status,verbs=get;update;patch // +kubebuilder:rbac:groups=operator.containers.carbonblack.io,resources=cbcontainersagents/finalizers,verbs=update -// +kubebuilder:rbac:groups=scheduling.k8s.io,resources=priorityclasses,verbs=* +// +kubebuilder:rbac:groups=scheduling.k8s.io,resources=priorityclasses,verbs=delete;get;patch;update,resourceNames=cbcontainers-dataplane-priority-class +// +kubebuilder:rbac:groups=scheduling.k8s.io,resources=priorityclasses,verbs=create;list;watch // +kubebuilder:rbac:groups=admissionregistration.k8s.io,resources={validatingwebhookconfigurations,mutatingwebhookconfigurations},verbs=* // +kubebuilder:rbac:groups={core},resources={nodes},verbs=list // +kubebuilder:rbac:groups={core},resources={namespaces},verbs=get From aafec7a70001f498ffcbe89c12f06130692441c4 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Mon, 3 Jul 2023 11:11:37 +0300 Subject: [PATCH 11/22] Restrict webhook RBAC by resource name when possible --- config/rbac/role.yaml | 16 +++++++++++++++- controllers/cbcontainersagent_controller.go | 4 +++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 50adeb80..b3ad61ea 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -12,7 +12,21 @@ rules: - mutatingwebhookconfigurations - validatingwebhookconfigurations verbs: - - '*' + - create + - list + - watch +- apiGroups: + - admissionregistration.k8s.io + resourceNames: + - cbcontainers-hardening-enforcer + resources: + - mutatingwebhookconfigurations + - validatingwebhookconfigurations + verbs: + - delete + - get + - patch + - update - apiGroups: - "" resources: diff --git a/controllers/cbcontainersagent_controller.go b/controllers/cbcontainersagent_controller.go index 4f5fb2fc..496895b8 100644 --- a/controllers/cbcontainersagent_controller.go +++ b/controllers/cbcontainersagent_controller.go @@ -76,13 +76,15 @@ func (r *CBContainersAgentController) getContainersAgentObject(ctx context.Conte // The following values must be kept in-sync with constants for generated RBAC to work properly: // - default dataplane namespace (see common.DataPlaneNamespaceName) // - cluster-wide dataplane priority class (see common.DataPlanePriorityClassName) - avoids access to all priority classes on the cluster +// - the cluster-wide webhooks (see components.EnforcerName) - avoids access to all webhooks on the cluster // +kubebuilder:rbac:groups=operator.containers.carbonblack.io,resources=cbcontainersagents,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=operator.containers.carbonblack.io,resources=cbcontainersagents/status,verbs=get;update;patch // +kubebuilder:rbac:groups=operator.containers.carbonblack.io,resources=cbcontainersagents/finalizers,verbs=update // +kubebuilder:rbac:groups=scheduling.k8s.io,resources=priorityclasses,verbs=delete;get;patch;update,resourceNames=cbcontainers-dataplane-priority-class // +kubebuilder:rbac:groups=scheduling.k8s.io,resources=priorityclasses,verbs=create;list;watch -// +kubebuilder:rbac:groups=admissionregistration.k8s.io,resources={validatingwebhookconfigurations,mutatingwebhookconfigurations},verbs=* +// +kubebuilder:rbac:groups=admissionregistration.k8s.io,resources={validatingwebhookconfigurations,mutatingwebhookconfigurations},verbs=delete;get;patch;update,resourceNames=cbcontainers-hardening-enforcer +// +kubebuilder:rbac:groups=admissionregistration.k8s.io,resources={validatingwebhookconfigurations,mutatingwebhookconfigurations},verbs=create;list;watch // +kubebuilder:rbac:groups={core},resources={nodes},verbs=list // +kubebuilder:rbac:groups={core},resources={namespaces},verbs=get // +kubebuilder:rbac:groups={policy},resources={podsecuritypolicies},verbs=use,resourceNames={cbcontainers-manager-psp} From 4aa8c4a79d69bd48b7fe6c8be45c79a7ac248665 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Mon, 3 Jul 2023 15:03:53 +0300 Subject: [PATCH 12/22] Move dataplane RBAC objects to the operator chart - so the agent chart mimics deploying a CR with secret --- .../templates/dataplane_rbac.yaml} | 0 .../templates/dataplane_service_accounts.yaml} | 12 ++++++------ 2 files changed, 6 insertions(+), 6 deletions(-) rename charts/{cbcontainers-agent/cbcontainers-agent-chart/templates/rbac.yaml => cbcontainers-operator/cbcontainers-operator-chart/templates/dataplane_rbac.yaml} (100%) rename charts/{cbcontainers-agent/cbcontainers-agent-chart/templates/service-accounts.yaml => cbcontainers-operator/cbcontainers-operator-chart/templates/dataplane_service_accounts.yaml} (66%) diff --git a/charts/cbcontainers-agent/cbcontainers-agent-chart/templates/rbac.yaml b/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/dataplane_rbac.yaml similarity index 100% rename from charts/cbcontainers-agent/cbcontainers-agent-chart/templates/rbac.yaml rename to charts/cbcontainers-operator/cbcontainers-operator-chart/templates/dataplane_rbac.yaml diff --git a/charts/cbcontainers-agent/cbcontainers-agent-chart/templates/service-accounts.yaml b/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/dataplane_service_accounts.yaml similarity index 66% rename from charts/cbcontainers-agent/cbcontainers-agent-chart/templates/service-accounts.yaml rename to charts/cbcontainers-operator/cbcontainers-operator-chart/templates/dataplane_service_accounts.yaml index 401f4548..3959f0ab 100644 --- a/charts/cbcontainers-agent/cbcontainers-agent-chart/templates/service-accounts.yaml +++ b/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/dataplane_service_accounts.yaml @@ -4,7 +4,7 @@ imagePullSecrets: kind: ServiceAccount metadata: name: cbcontainers-agent-node - namespace: {{ default "cbcontainers-dataplane" .Values.agentNamespace }} + namespace: {{ default "cbcontainers-dataplane" .Values.operatorNamespace }} --- apiVersion: v1 imagePullSecrets: @@ -12,7 +12,7 @@ imagePullSecrets: kind: ServiceAccount metadata: name: cbcontainers-enforcer - namespace: {{ default "cbcontainers-dataplane" .Values.agentNamespace }} + namespace: {{ default "cbcontainers-dataplane" .Values.operatorNamespace }} --- apiVersion: v1 imagePullSecrets: @@ -20,7 +20,7 @@ imagePullSecrets: kind: ServiceAccount metadata: name: cbcontainers-image-scanning - namespace: {{ default "cbcontainers-dataplane" .Values.agentNamespace }} + namespace: {{ default "cbcontainers-dataplane" .Values.operatorNamespace }} --- apiVersion: v1 imagePullSecrets: @@ -28,7 +28,7 @@ imagePullSecrets: kind: ServiceAccount metadata: name: cbcontainers-monitor - namespace: {{ default "cbcontainers-dataplane" .Values.agentNamespace }} + namespace: {{ default "cbcontainers-dataplane" .Values.operatorNamespace }} --- apiVersion: v1 imagePullSecrets: @@ -36,7 +36,7 @@ imagePullSecrets: kind: ServiceAccount metadata: name: cbcontainers-runtime-resolver - namespace: {{ default "cbcontainers-dataplane" .Values.agentNamespace }} + namespace: {{ default "cbcontainers-dataplane" .Values.operatorNamespace }} --- apiVersion: v1 imagePullSecrets: @@ -44,4 +44,4 @@ imagePullSecrets: kind: ServiceAccount metadata: name: cbcontainers-state-reporter - namespace: {{ default "cbcontainers-dataplane" .Values.agentNamespace }} + namespace: {{ default "cbcontainers-dataplane" .Values.operatorNamespace }} From 02b1a5389ff0c9a3a2cfbf6d5fe329ab47489fc3 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Mon, 3 Jul 2023 15:04:44 +0300 Subject: [PATCH 13/22] Update Readmes to match the "same-namespace" enforcement. Remove references to repositories that we don't support. Added the missing labeling when pre-creating a namespace. --- charts/cbcontainers-agent/README.md | 19 +++++++--------- charts/cbcontainers-operator/README.md | 31 +++++++++++++++----------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/charts/cbcontainers-agent/README.md b/charts/cbcontainers-agent/README.md index 2b654b34..aca3cf7a 100644 --- a/charts/cbcontainers-agent/README.md +++ b/charts/cbcontainers-agent/README.md @@ -21,15 +21,7 @@ There are 8 required fields that need to be provided by the user: | `spec.gateways.hardeningEventsGatewayHost` | The URL of the CBC Hardening events Gateway | | `spec.gateways.runtimeEventsGatewayHost` | The URL of the CBC Runtime events Gateway | -After setting these required fields in a `values.yaml` file you can install the chart from our repo: - -```sh -helm repo add vmware TODO-chart-repo/TODO-chart-name -f values.yaml -helm repo update -helm install cbcontainers-agent TODO-chart-repo/TODO-chart-name -f values.yaml --namespace cbcontainers-dataplane -``` - -or from source +After setting these required fields in a `values.yaml` file you can install the chart from source ```sh cd charts/cbcontainers-agent @@ -46,9 +38,14 @@ For all the possible values see +kubectl create namespace $NAMESPACE +kubectl label namespace $NAMESPACE control-plane=operator octarine=ignore +helm install cbcontainers-operator ./cbcontainers-operator-chart --set createOperatorNamespace=false,operatorNamespace=$NAMESPACE +``` + ### CRD Installation By default, installing the chart will also create the `CBContainersAgent` CRD. @@ -71,7 +73,7 @@ For more info see Date: Tue, 4 Jul 2023 11:15:29 +0300 Subject: [PATCH 14/22] Sync operator.yaml in the chart with RBAC changes and some missing items from last releases --- .../templates/operator.yaml | 249 +++++++++++++----- 1 file changed, 178 insertions(+), 71 deletions(-) diff --git a/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml b/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml index 98f26019..157a7305 100644 --- a/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml +++ b/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml @@ -1846,34 +1846,17 @@ spec: default: {} properties: CRIO: - default: { } - description: CRIO holds configuration values specific - to the CRI-O container engine + default: {} + description: CRIO holds configuration values specific to the CRI-O container engine properties: configPath: - description: ConfigPath can be used to set the - path to CRI-O's configuration file. If not specified, - the default location is used (/etc/crio/crio.conf). - See https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md - for more information. + description: ConfigPath can be used to set the path to CRI-O's configuration file. If not specified, the default location is used (/etc/crio/crio.conf). See https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md for more information. type: string storageConfigPath: - description: StorageConfigPath can be used to - set the path to the storage configuration file - used by CRI-O (if any). If not specified, the - default location for storage is used (/etc/containers/storage.conf). - The files does not need to exist. See https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md - for more information + description: StorageConfigPath can be used to set the path to the storage configuration file used by CRI-O (if any). If not specified, the default location for storage is used (/etc/containers/storage.conf). The files does not need to exist. See https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md for more information type: string storagePath: - description: StoragePath can be used to set the - path used by CRI-O to store images on each node. - This path will be mounted on the cluster scanner - to provide access to the node's images. If the - path does not match what CRI-O uses on the nodes, - then images will not be found and scanned as - expected. If not specified, the default location - of CRI-O is used (/var/lib/containers/storage). + description: StoragePath can be used to set the path used by CRI-O to store images on each node. This path will be mounted on the cluster scanner to provide access to the node's images. If the path does not match what CRI-O uses on the nodes, then images will not be found and scanned as expected. If not specified, the default location of CRI-O is used (/var/lib/containers/storage). type: string type: object endpoint: @@ -2558,6 +2541,103 @@ spec: type: object type: object type: object + cndr: + description: CBContainersCndrSpec defines the desired state of CBContainersCndr + properties: + companyCodeSecretName: + default: cbcontainers-company-code + type: string + enabled: + default: false + type: boolean + sensor: + default: {} + properties: + daemonSetAnnotations: + additionalProperties: + type: string + default: {} + type: object + env: + additionalProperties: + type: string + default: {} + type: object + image: + default: + repository: cbartifactory/cndr + properties: + pullPolicy: + default: IfNotPresent + description: PullPolicy describes a policy for if/when to pull a container image + type: string + pullSecrets: + description: "PullSecrets is a list of secret names, which will be used to pull the container image(s). \n The secrets must already exist." + items: + type: string + type: array + repository: + type: string + tag: + type: string + type: object + labels: + additionalProperties: + type: string + default: {} + type: object + logLevel: + default: info + type: string + podTemplateAnnotations: + additionalProperties: + type: string + default: {} + type: object + prometheus: + default: + port: 7071 + properties: + enabled: + default: false + type: boolean + port: + type: integer + type: object + resources: + default: + limits: + cpu: 500m + memory: 1024Mi + requests: + cpu: 30m + memory: 64Mi + description: ResourceRequirements describes the compute resource requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + verbosity_level: + default: 2 + type: integer + type: object + type: object runtimeProtection: default: {} description: CBContainersRuntimeProtectionSpec defines the desired state of CBContainersRuntime @@ -3075,6 +3155,10 @@ spec: type: string default: {} type: object + nodesToReplicasRatio: + default: 5 + format: int32 + type: integer podTemplateAnnotations: additionalProperties: type: string @@ -3128,7 +3212,6 @@ spec: type: integer type: object replicasCount: - default: 1 format: int32 type: integer resources: @@ -3381,6 +3464,7 @@ spec: type: object namespace: default: cbcontainers-dataplane + description: Namespace is deprecated and the value has no effect. Do not use. type: string version: type: string @@ -3472,87 +3556,82 @@ rules: - patch --- apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole +kind: Role metadata: creationTimestamp: null name: cbcontainers-manager-role + namespace: {{ default "cbcontainers-dataplane" .Values.operatorNamespace }} rules: - apiGroups: - - "" - - discovery.k8s.io - resources: - - endpoints - - endpointslices - - services - verbs: - - get - - list - - watch -- apiGroups: - - admissionregistration.k8s.io - resources: - - mutatingwebhookconfigurations - - validatingwebhookconfigurations - verbs: - - '*' -- apiGroups: - - apiextensions.k8s.io - apps - - batch - "" - - extensions - - networking.k8s.io - - rbac - - rbac.authorization.k8s.io resources: - - clusterrolebindings - - cronjobs - - customresourcedefinitions - daemonsets - deployments - - ingresses - - jobs - - namespaces - - networkpolicies - - nodes - - pods - - replicasets - - replicationcontrollers - - rolebindings - services - - statefulsets verbs: + - create + - delete + - deletecollection - get - list + - patch + - update - watch - apiGroups: - - apps - "" resources: - - daemonsets - - deployments - - services + - configmaps + - secrets verbs: - create - delete + - deletecollection - get - list - patch - update - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: cbcontainers-manager-role +rules: - apiGroups: - - "" + - admissionregistration.k8s.io resources: - - configmaps - - secrets + - mutatingwebhookconfigurations + - validatingwebhookconfigurations verbs: - create + - list + - watch +- apiGroups: + - admissionregistration.k8s.io + resourceNames: + - cbcontainers-hardening-enforcer + resources: + - mutatingwebhookconfigurations + - validatingwebhookconfigurations + verbs: - delete - get - - list - patch - update - - watch +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get +- apiGroups: + - "" + resources: + - nodes + verbs: + - list - apiGroups: - operator.containers.carbonblack.io resources: @@ -3592,7 +3671,20 @@ rules: resources: - priorityclasses verbs: - - '*' + - create + - list + - watch +- apiGroups: + - scheduling.k8s.io + resourceNames: + - cbcontainers-dataplane-priority-class + resources: + - priorityclasses + verbs: + - delete + - get + - patch + - update --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -3627,6 +3719,20 @@ subjects: namespace: {{ default "cbcontainers-dataplane" .Values.operatorNamespace }} --- apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cbcontainers-manager-namespaced-rolebinding + namespace: {{ default "cbcontainers-dataplane" .Values.operatorNamespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cbcontainers-manager-role +subjects: +- kind: ServiceAccount + name: cbcontainers-operator + namespace: {{ default "cbcontainers-dataplane" .Values.operatorNamespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: cbcontainers-manager-rolebinding @@ -3691,6 +3797,7 @@ spec: ports: - name: https port: 8443 + protocol: TCP targetPort: https selector: control-plane: operator From f7b05fd415d082b1dfcab9b60912d4ebc23e5a74 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Tue, 4 Jul 2023 11:16:37 +0300 Subject: [PATCH 15/22] Bump charts version due to breaking changes. Bumped app version there as well to be latest so far. --- charts/cbcontainers-agent/cbcontainers-agent-chart/Chart.yaml | 4 ++-- .../cbcontainers-operator-chart/Chart.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/cbcontainers-agent/cbcontainers-agent-chart/Chart.yaml b/charts/cbcontainers-agent/cbcontainers-agent-chart/Chart.yaml index 43a0ac63..f32f96ca 100644 --- a/charts/cbcontainers-agent/cbcontainers-agent-chart/Chart.yaml +++ b/charts/cbcontainers-agent/cbcontainers-agent-chart/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: cbcontainers-agent description: A Helm chart for installing the CBContainers Agent type: application -version: 1.0.0 -appVersion: "2.11.0" +version: 2.0.0 +appVersion: "2.12.1" diff --git a/charts/cbcontainers-operator/cbcontainers-operator-chart/Chart.yaml b/charts/cbcontainers-operator/cbcontainers-operator-chart/Chart.yaml index 5b890cab..9e58d581 100644 --- a/charts/cbcontainers-operator/cbcontainers-operator-chart/Chart.yaml +++ b/charts/cbcontainers-operator/cbcontainers-operator-chart/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: cbcontainers-operator description: A Helm chart for installing the CBContainers operator type: application -version: 1.0.0 -appVersion: v5.6.0 +version: 2.0.0 +appVersion: v5.6.2 From 6e5d7bc577ef1911de5cf87a8ed64cdd3616cde3 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Tue, 4 Jul 2023 11:17:39 +0300 Subject: [PATCH 16/22] Minor comment change --- charts/cbcontainers-agent/cbcontainers-agent-chart/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/cbcontainers-agent/cbcontainers-agent-chart/values.yaml b/charts/cbcontainers-agent/cbcontainers-agent-chart/values.yaml index dad9ba86..91c04a30 100644 --- a/charts/cbcontainers-agent/cbcontainers-agent-chart/values.yaml +++ b/charts/cbcontainers-agent/cbcontainers-agent-chart/values.yaml @@ -9,5 +9,5 @@ clusterGroup: "" # clusterName is the name that will be used for the cluster that the agent is installed on clusterName: "" # agentNamespace is the name of the namespace in which the agent will be installed -# that namespace should exist before the chart is installed +# that namespace must exist before the chart is installed and must match the namespace where the operator is deployed agentNamespace: "cbcontainers-dataplane" \ No newline at end of file From d56ec34e2b9a413d60d1117044503d664d6939db Mon Sep 17 00:00:00 2001 From: ltsonov Date: Tue, 4 Jul 2023 12:07:44 +0300 Subject: [PATCH 17/22] Added some explanation behind the RBAC setup and how to update it --- docs/rbac.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 docs/rbac.md diff --git a/docs/rbac.md b/docs/rbac.md new file mode 100644 index 00000000..397b1da9 --- /dev/null +++ b/docs/rbac.md @@ -0,0 +1,22 @@ +# How to configure and use the VMWare Carbon Black Cloud Container Operator's Role-based access control (RBAC) + +## Design for the operator and agent RBAC definition +Following the principle of least-privilege, any permission given to the operator should have good reason and be scoped as tightly as possible. + +In practice, this means: +* If the resource is namespaced and part of the agent, use a `Role` to give permissions in the agent's namespace only +* If the resource is namespaced and not part of the agent, + * and you need to read it - use a `ClusterRole` unless you are 100% sure what the namespace will be + * and you need to modify it - do you _really_ need to? +* If the resource is non-namespaced, use a `ClusterRole` and try to restrict `delete,get,update,patch` via `resourceNames` (`create, list, watch` either don't support this restriction or require extra care) + +## Changing the operator's access levels +These permissions are generated by `controller-gen` and controlled via `+kubebuilder` directives. See [the controller definitions](../controllers/cbcontainersagent_controller.go). +Any change to those directives requires running `make manifests` to update the respective [`role.yaml`](../config/rbac/role.yaml) file. Changes should also be propagated to the helm charts as well. + + +## Changing the agent components access levels +These are maintained manually in [dataplane_roles.yaml](../config/rbac/dataplane/dataplane_roles.yaml) and [the helm equivalent](../charts/cbcontainers-operator/cbcontainers-operator-chart/templates/dataplane_rbac.yaml). +Same goes for the service accounts and role bindings. Changes should be applied in both places. + +The roles should follow the least-privilege principle, same as the operator. Note that the agent components often need _more_ permissions than the operator to work as expected. \ No newline at end of file From 761070d1a77fdf1e5bd45524435e046955d69841 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Tue, 4 Jul 2023 13:13:24 +0300 Subject: [PATCH 18/22] Update deployment.yaml to also mount the namespace env var --- .../cbcontainers-operator-chart/templates/deployment.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/deployment.yaml b/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/deployment.yaml index a408648c..43a21c51 100644 --- a/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/deployment.yaml +++ b/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/deployment.yaml @@ -99,5 +99,10 @@ spec: readOnlyRootFilesystem: true runAsNonRoot: true runAsUser: 65532 + env: + - name: OPERATOR_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace serviceAccountName: cbcontainers-operator terminationGracePeriodSeconds: 10 From 0b247de7aef5cd7f2f8379f1e3a2ed9dab61d579 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Tue, 4 Jul 2023 13:28:24 +0300 Subject: [PATCH 19/22] Replace namespace var in dataplane_rbac.yaml --- .../templates/dataplane_rbac.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/dataplane_rbac.yaml b/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/dataplane_rbac.yaml index f86348da..3f56e6b3 100644 --- a/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/dataplane_rbac.yaml +++ b/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/dataplane_rbac.yaml @@ -115,7 +115,7 @@ roleRef: subjects: - kind: ServiceAccount name: cbcontainers-agent-node - namespace: {{ default "cbcontainers-dataplane" .Values.agentNamespace }} + namespace: {{ default "cbcontainers-dataplane" .Values.operatorNamespace }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -128,7 +128,7 @@ roleRef: subjects: - kind: ServiceAccount name: cbcontainers-enforcer - namespace: {{ default "cbcontainers-dataplane" .Values.agentNamespace }} + namespace: {{ default "cbcontainers-dataplane" .Values.operatorNamespace }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -141,7 +141,7 @@ roleRef: subjects: - kind: ServiceAccount name: cbcontainers-image-scanning - namespace: {{ default "cbcontainers-dataplane" .Values.agentNamespace }} + namespace: {{ default "cbcontainers-dataplane" .Values.operatorNamespace }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -154,7 +154,7 @@ roleRef: subjects: - kind: ServiceAccount name: cbcontainers-monitor - namespace: {{ default "cbcontainers-dataplane" .Values.agentNamespace }} + namespace: {{ default "cbcontainers-dataplane" .Values.operatorNamespace }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -167,7 +167,7 @@ roleRef: subjects: - kind: ServiceAccount name: cbcontainers-runtime-resolver - namespace: {{ default "cbcontainers-dataplane" .Values.agentNamespace }} + namespace: {{ default "cbcontainers-dataplane" .Values.operatorNamespace }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -180,4 +180,4 @@ roleRef: subjects: - kind: ServiceAccount name: cbcontainers-state-reporter - namespace: {{ default "cbcontainers-dataplane" .Values.agentNamespace }} + namespace: {{ default "cbcontainers-dataplane" .Values.operatorNamespace }} From 5c0c648caaf66f96647b46e8a40369ebf620982f Mon Sep 17 00:00:00 2001 From: ltsonov Date: Tue, 11 Jul 2023 15:37:05 +0300 Subject: [PATCH 20/22] Added deprecation notice as godoc to CRD.Namespace --- api/v1/cbcontainersagent_types.go | 1 + ...perator.containers.carbonblack.io_cbcontainersagents.yaml | 5 +++-- ...perator.containers.carbonblack.io_cbcontainersagents.yaml | 5 +++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/api/v1/cbcontainersagent_types.go b/api/v1/cbcontainersagent_types.go index 4dca372e..e3ae0c59 100644 --- a/api/v1/cbcontainersagent_types.go +++ b/api/v1/cbcontainersagent_types.go @@ -36,6 +36,7 @@ type CBContainersAgentSpec struct { // +kubebuilder:default:="cbcontainers-dataplane" // Namespace is deprecated and the value has no effect. Do not use. + // Deprecated: The operator and agent always run in the same namespace. See documentation for ways to customize this namespace. Namespace string `json:"namespace,omitempty"` // +kubebuilder:default:="cbcontainers-access-token" AccessTokenSecretName string `json:"accessTokenSecretName,omitempty"` diff --git a/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml b/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml index ed0b9ed6..f2733d5e 100644 --- a/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml +++ b/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml @@ -6036,8 +6036,9 @@ spec: type: object namespace: default: cbcontainers-dataplane - description: Namespace is deprecated and the value has no effect. - Do not use. + description: 'Namespace is deprecated and the value has no effect. + Do not use. Deprecated: The operator and agent always run in the + same namespace. See documentation for ways to customize this namespace.' type: string version: type: string diff --git a/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml b/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml index f7eb8a3c..2bc77744 100644 --- a/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml +++ b/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml @@ -5608,8 +5608,9 @@ spec: - runtimeEventsGateway type: object namespace: - description: Namespace is deprecated and the value has no effect. Do - not use. + description: 'Namespace is deprecated and the value has no effect. Do + not use. Deprecated: The operator and agent always run in the same + namespace. See documentation for ways to customize this namespace.' type: string version: type: string From 60f0303d2e70a4350071f62267ff8318000cc864 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Tue, 11 Jul 2023 16:11:43 +0300 Subject: [PATCH 21/22] Add Namespace to each component's constructor. Don't set the namespace on the k8s object as it has no affect (NamespacedName is what sets the namespace). --- .../state/components/cluster_configmap.go | 7 ++- .../state/components/enforcer_deployment.go | 5 +- .../components/enforcer_mutating_webhook.go | 4 +- .../state/components/enforcer_service.go | 6 +-- .../state/components/enforcer_tls_secret.go | 6 +-- .../components/enforcer_validating_webhook.go | 5 +- .../image_scanning_reporter_deployment.go | 5 +- .../image_scanning_reporter_service.go | 6 +-- .../state/components/monitor_deployment.go | 5 +- .../state/components/registry_secret.go | 5 +- .../state/components/resolver_service.go | 6 +-- .../components/runtime_resolver_deployment.go | 5 +- .../state/components/sensor_daemon_set.go | 5 +- .../components/state_reporter_deployment.go | 5 +- cbcontainers/state/state_applier.go | 49 ++++++------------- 15 files changed, 43 insertions(+), 81 deletions(-) diff --git a/cbcontainers/state/components/cluster_configmap.go b/cbcontainers/state/components/cluster_configmap.go index be8bea39..155cd218 100644 --- a/cbcontainers/state/components/cluster_configmap.go +++ b/cbcontainers/state/components/cluster_configmap.go @@ -17,9 +17,9 @@ type ConfigurationK8sObject struct { Namespace string } -func NewConfigurationK8sObject() *ConfigurationK8sObject { +func NewConfigurationK8sObject(namespace string) *ConfigurationK8sObject { return &ConfigurationK8sObject{ - Namespace: commonState.DataPlaneNamespaceName, + Namespace: namespace, } } @@ -35,12 +35,11 @@ func (obj *ConfigurationK8sObject) MutateK8sObject(k8sObject client.Object, agen return fmt.Errorf("expected ConfigMap K8s object") } - configMap.Namespace = obj.Namespace configMap.Data = map[string]string{ commonState.DataPlaneConfigmapAccountKey: agentSpec.Account, commonState.DataPlaneConfigmapClusterKey: agentSpec.ClusterName, commonState.DataPlaneConfigmapAgentVersionKey: agentSpec.Version, - commonState.DataPlaneConfigmapDataplaneNamespaceKey: configMap.Namespace, + commonState.DataPlaneConfigmapDataplaneNamespaceKey: obj.Namespace, commonState.DataPlaneConfigmapApiSchemeKey: agentSpec.Gateways.ApiGateway.Scheme, commonState.DataPlaneConfigmapApiHostKey: agentSpec.Gateways.ApiGateway.Host, commonState.DataPlaneConfigmapApiPortKey: strconv.Itoa(agentSpec.Gateways.ApiGateway.Port), diff --git a/cbcontainers/state/components/enforcer_deployment.go b/cbcontainers/state/components/enforcer_deployment.go index f20c13df..5e629d62 100644 --- a/cbcontainers/state/components/enforcer_deployment.go +++ b/cbcontainers/state/components/enforcer_deployment.go @@ -42,9 +42,9 @@ type EnforcerDeploymentK8sObject struct { Namespace string } -func NewEnforcerDeploymentK8sObject() *EnforcerDeploymentK8sObject { +func NewEnforcerDeploymentK8sObject(namespace string) *EnforcerDeploymentK8sObject { return &EnforcerDeploymentK8sObject{ - Namespace: commonState.DataPlaneNamespaceName, + Namespace: namespace, } } @@ -84,7 +84,6 @@ func (obj *EnforcerDeploymentK8sObject) MutateK8sObject(k8sObject client.Object, if objectsDiffer(deployment.Spec.Template.Spec.ImagePullSecrets, desiredImagePullSecrets) { deployment.Spec.Template.Spec.ImagePullSecrets = desiredImagePullSecrets } - deployment.Namespace = obj.Namespace obj.mutateAnnotations(deployment, enforcer) obj.mutateVolumes(&deployment.Spec.Template.Spec) obj.mutateAffinityAndNodeSelector(&deployment.Spec.Template.Spec, enforcer) diff --git a/cbcontainers/state/components/enforcer_mutating_webhook.go b/cbcontainers/state/components/enforcer_mutating_webhook.go index dca99af4..44c5085e 100644 --- a/cbcontainers/state/components/enforcer_mutating_webhook.go +++ b/cbcontainers/state/components/enforcer_mutating_webhook.go @@ -34,10 +34,10 @@ type EnforcerMutatingWebhookK8sObject struct { ServiceNamespace string } -func NewEnforcerMutatingWebhookK8sObject(kubeletVersion string) *EnforcerMutatingWebhookK8sObject { +func NewEnforcerMutatingWebhookK8sObject(serviceNamespace, kubeletVersion string) *EnforcerMutatingWebhookK8sObject { return &EnforcerMutatingWebhookK8sObject{ kubeletVersion: kubeletVersion, - ServiceNamespace: commonState.DataPlaneNamespaceName, + ServiceNamespace: serviceNamespace, } } diff --git a/cbcontainers/state/components/enforcer_service.go b/cbcontainers/state/components/enforcer_service.go index 3cde3e13..21819796 100644 --- a/cbcontainers/state/components/enforcer_service.go +++ b/cbcontainers/state/components/enforcer_service.go @@ -4,7 +4,6 @@ import ( "fmt" cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" - commonState "github.com/vmware/cbcontainers-operator/cbcontainers/state/common" coreV1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" @@ -21,9 +20,9 @@ type EnforcerServiceK8sObject struct { Namespace string } -func NewEnforcerServiceK8sObject() *EnforcerServiceK8sObject { +func NewEnforcerServiceK8sObject(namespace string) *EnforcerServiceK8sObject { return &EnforcerServiceK8sObject{ - Namespace: commonState.DataPlaneNamespaceName, + Namespace: namespace, } } @@ -45,7 +44,6 @@ func (obj *EnforcerServiceK8sObject) MutateK8sObject(k8sObject client.Object, ag service.Labels = enforcer.Labels service.Spec.Type = coreV1.ServiceTypeClusterIP - service.Namespace = obj.Namespace service.Spec.Selector = map[string]string{ EnforcerLabelKey: EnforcerName, } diff --git a/cbcontainers/state/components/enforcer_tls_secret.go b/cbcontainers/state/components/enforcer_tls_secret.go index f9b03cef..b4bed8bd 100644 --- a/cbcontainers/state/components/enforcer_tls_secret.go +++ b/cbcontainers/state/components/enforcer_tls_secret.go @@ -5,7 +5,6 @@ import ( cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" "github.com/vmware/cbcontainers-operator/cbcontainers/models" - commonState "github.com/vmware/cbcontainers-operator/cbcontainers/state/common" coreV1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -26,10 +25,10 @@ type EnforcerTlsK8sObject struct { Namespace string } -func NewEnforcerTlsK8sObject(tlsSecretsValuesCreator TlsSecretsValuesCreator) *EnforcerTlsK8sObject { +func NewEnforcerTlsK8sObject(namespace string, tlsSecretsValuesCreator TlsSecretsValuesCreator) *EnforcerTlsK8sObject { return &EnforcerTlsK8sObject{ tlsSecretsValuesCreator: tlsSecretsValuesCreator, - Namespace: commonState.DataPlaneNamespaceName, + Namespace: namespace, } } @@ -47,7 +46,6 @@ func (obj *EnforcerTlsK8sObject) MutateK8sObject(k8sObject client.Object, spec * return fmt.Errorf("expected Secret K8s object") } - secret.Namespace = obj.Namespace tlsSecretValues, err := obj.tlsSecretsValuesCreator.CreateTlsSecretsValues(types.NamespacedName{Name: EnforcerName, Namespace: obj.Namespace}) if err != nil { return err diff --git a/cbcontainers/state/components/enforcer_validating_webhook.go b/cbcontainers/state/components/enforcer_validating_webhook.go index 1e855f1e..acaeaf14 100644 --- a/cbcontainers/state/components/enforcer_validating_webhook.go +++ b/cbcontainers/state/components/enforcer_validating_webhook.go @@ -7,7 +7,6 @@ import ( cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" "github.com/vmware/cbcontainers-operator/cbcontainers/models" "github.com/vmware/cbcontainers-operator/cbcontainers/state/adapters" - commonState "github.com/vmware/cbcontainers-operator/cbcontainers/state/common" "github.com/vmware/cbcontainers-operator/cbcontainers/utils" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -36,10 +35,10 @@ type EnforcerValidatingWebhookK8sObject struct { ServiceNamespace string } -func NewEnforcerValidatingWebhookK8sObject(kubeletVersion string) *EnforcerValidatingWebhookK8sObject { +func NewEnforcerValidatingWebhookK8sObject(serviceNamespace, kubeletVersion string) *EnforcerValidatingWebhookK8sObject { return &EnforcerValidatingWebhookK8sObject{ kubeletVersion: kubeletVersion, - ServiceNamespace: commonState.DataPlaneNamespaceName, + ServiceNamespace: serviceNamespace, } } diff --git a/cbcontainers/state/components/image_scanning_reporter_deployment.go b/cbcontainers/state/components/image_scanning_reporter_deployment.go index 3696ba75..4d9b25d6 100644 --- a/cbcontainers/state/components/image_scanning_reporter_deployment.go +++ b/cbcontainers/state/components/image_scanning_reporter_deployment.go @@ -34,9 +34,9 @@ type ImageScanningReporterDeploymentK8sObject struct { Namespace string } -func NewImageScanningReporterDeploymentK8sObject() *ImageScanningReporterDeploymentK8sObject { +func NewImageScanningReporterDeploymentK8sObject(namespace string) *ImageScanningReporterDeploymentK8sObject { return &ImageScanningReporterDeploymentK8sObject{ - Namespace: commonState.DataPlaneNamespaceName, + Namespace: namespace, } } @@ -55,7 +55,6 @@ func (obj *ImageScanningReporterDeploymentK8sObject) MutateK8sObject(k8sObject c } clusterScanning := &agentSpec.Components.ClusterScanning - deployment.Namespace = obj.Namespace imageScanningReporter := &clusterScanning.ImageScanningReporter obj.initiateDeployment(deployment, agentSpec) obj.mutateLabels(deployment, imageScanningReporter) diff --git a/cbcontainers/state/components/image_scanning_reporter_service.go b/cbcontainers/state/components/image_scanning_reporter_service.go index 68501329..2f7dbce6 100644 --- a/cbcontainers/state/components/image_scanning_reporter_service.go +++ b/cbcontainers/state/components/image_scanning_reporter_service.go @@ -4,7 +4,6 @@ import ( "fmt" cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" - commonState "github.com/vmware/cbcontainers-operator/cbcontainers/state/common" coreV1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" @@ -21,9 +20,9 @@ type ImageScanningReporterServiceK8sObject struct { Namespace string } -func NewImageScanningReporterServiceK8sObject() *ImageScanningReporterServiceK8sObject { +func NewImageScanningReporterServiceK8sObject(namespace string) *ImageScanningReporterServiceK8sObject { return &ImageScanningReporterServiceK8sObject{ - Namespace: commonState.DataPlaneNamespaceName, + Namespace: namespace, } } @@ -43,7 +42,6 @@ func (obj *ImageScanningReporterServiceK8sObject) MutateK8sObject(k8sObject clie imageScanningReporter := &agentSpec.Components.ClusterScanning.ImageScanningReporter - service.Namespace = obj.Namespace service.Labels = imageScanningReporter.Labels service.Spec.Type = coreV1.ServiceTypeClusterIP service.Spec.Selector = map[string]string{ diff --git a/cbcontainers/state/components/monitor_deployment.go b/cbcontainers/state/components/monitor_deployment.go index 4a757b48..809895ba 100644 --- a/cbcontainers/state/components/monitor_deployment.go +++ b/cbcontainers/state/components/monitor_deployment.go @@ -37,9 +37,9 @@ type MonitorDeploymentK8sObject struct { Namespace string } -func NewMonitorDeploymentK8sObject() *MonitorDeploymentK8sObject { +func NewMonitorDeploymentK8sObject(namespace string) *MonitorDeploymentK8sObject { return &MonitorDeploymentK8sObject{ - Namespace: commonState.DataPlaneNamespaceName, + Namespace: namespace, } } @@ -76,7 +76,6 @@ func (obj *MonitorDeploymentK8sObject) MutateK8sObject(k8sObject client.Object, deployment.Spec.Template.ObjectMeta.Annotations = make(map[string]string) } - deployment.Namespace = obj.Namespace deployment.Spec.Replicas = &MonitorReplicas deployment.ObjectMeta.Labels = desiredLabels deployment.Spec.Selector.MatchLabels = desiredLabels diff --git a/cbcontainers/state/components/registry_secret.go b/cbcontainers/state/components/registry_secret.go index 2fd14250..845ef397 100644 --- a/cbcontainers/state/components/registry_secret.go +++ b/cbcontainers/state/components/registry_secret.go @@ -18,9 +18,9 @@ type RegistrySecretK8sObject struct { Namespace string } -func NewRegistrySecretK8sObject() *RegistrySecretK8sObject { +func NewRegistrySecretK8sObject(namespace string) *RegistrySecretK8sObject { return &RegistrySecretK8sObject{ - Namespace: commonState.DataPlaneNamespaceName, + Namespace: namespace, } } @@ -46,7 +46,6 @@ func (obj *RegistrySecretK8sObject) MutateK8sObject(k8sObject client.Object, spe secret.Type = obj.registrySecretValues.Type secret.Data = obj.registrySecretValues.Data - secret.Namespace = obj.Namespace return nil } diff --git a/cbcontainers/state/components/resolver_service.go b/cbcontainers/state/components/resolver_service.go index 9a321ad1..935889d6 100644 --- a/cbcontainers/state/components/resolver_service.go +++ b/cbcontainers/state/components/resolver_service.go @@ -4,7 +4,6 @@ import ( "fmt" cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" - commonState "github.com/vmware/cbcontainers-operator/cbcontainers/state/common" coreV1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" @@ -20,9 +19,9 @@ type ResolverServiceK8sObject struct { Namespace string } -func NewResolverServiceK8sObject() *ResolverServiceK8sObject { +func NewResolverServiceK8sObject(namespace string) *ResolverServiceK8sObject { return &ResolverServiceK8sObject{ - Namespace: commonState.DataPlaneNamespaceName, + Namespace: namespace, } } @@ -46,7 +45,6 @@ func (obj *ResolverServiceK8sObject) MutateK8sObject(k8sObject client.Object, ag service.Spec.Type = coreV1.ServiceTypeClusterIP service.Spec.ClusterIP = coreV1.ClusterIPNone - service.Namespace = obj.Namespace service.Spec.Selector = map[string]string{ resolverLabelKey: ResolverName, } diff --git a/cbcontainers/state/components/runtime_resolver_deployment.go b/cbcontainers/state/components/runtime_resolver_deployment.go index 53354ee7..9be0d68b 100644 --- a/cbcontainers/state/components/runtime_resolver_deployment.go +++ b/cbcontainers/state/components/runtime_resolver_deployment.go @@ -36,9 +36,9 @@ type ResolverDeploymentK8sObject struct { APIReader client.Reader } -func NewResolverDeploymentK8sObject(apiReader client.Reader) *ResolverDeploymentK8sObject { +func NewResolverDeploymentK8sObject(namespace string, apiReader client.Reader) *ResolverDeploymentK8sObject { return &ResolverDeploymentK8sObject{ - Namespace: commonState.DataPlaneNamespaceName, + Namespace: namespace, APIReader: apiReader, } } @@ -89,7 +89,6 @@ func (obj *ResolverDeploymentK8sObject) MutateK8sObject(k8sObject client.Object, } } - deployment.Namespace = obj.Namespace deployment.Spec.Replicas = replicasCount deployment.ObjectMeta.Labels = desiredLabels deployment.Spec.Selector.MatchLabels = desiredLabels diff --git a/cbcontainers/state/components/sensor_daemon_set.go b/cbcontainers/state/components/sensor_daemon_set.go index 13a16f1d..628d3f50 100644 --- a/cbcontainers/state/components/sensor_daemon_set.go +++ b/cbcontainers/state/components/sensor_daemon_set.go @@ -84,9 +84,9 @@ type SensorDaemonSetK8sObject struct { Namespace string } -func NewSensorDaemonSetK8sObject() *SensorDaemonSetK8sObject { +func NewSensorDaemonSetK8sObject(namespace string) *SensorDaemonSetK8sObject { return &SensorDaemonSetK8sObject{ - Namespace: commonState.DataPlaneNamespaceName, + Namespace: namespace, } } @@ -120,7 +120,6 @@ func (obj *SensorDaemonSetK8sObject) MutateK8sObject(k8sObject client.Object, ag daemonSet.Spec.Template.Spec.HostPID = false } - daemonSet.Namespace = obj.Namespace obj.mutateLabels(daemonSet, agentSpec) obj.mutateAnnotations(daemonSet, agentSpec) obj.mutateVolumes(daemonSet, agentSpec) diff --git a/cbcontainers/state/components/state_reporter_deployment.go b/cbcontainers/state/components/state_reporter_deployment.go index e49fbe27..b090b06d 100644 --- a/cbcontainers/state/components/state_reporter_deployment.go +++ b/cbcontainers/state/components/state_reporter_deployment.go @@ -32,9 +32,9 @@ type StateReporterDeploymentK8sObject struct { Namespace string } -func NewStateReporterDeploymentK8sObject() *StateReporterDeploymentK8sObject { +func NewStateReporterDeploymentK8sObject(namespace string) *StateReporterDeploymentK8sObject { return &StateReporterDeploymentK8sObject{ - Namespace: commonState.DataPlaneNamespaceName, + Namespace: namespace, } } @@ -72,7 +72,6 @@ func (obj *StateReporterDeploymentK8sObject) MutateK8sObject(k8sObject client.Ob deployment.Spec.Template.ObjectMeta.Annotations = make(map[string]string) } - deployment.Namespace = obj.Namespace deployment.Spec.Replicas = &StateReporterReplicas deployment.ObjectMeta.Labels = desiredLabels deployment.Spec.Selector.MatchLabels = desiredLabels diff --git a/cbcontainers/state/state_applier.go b/cbcontainers/state/state_applier.go index 6c597d50..35ebc408 100644 --- a/cbcontainers/state/state_applier.go +++ b/cbcontainers/state/state_applier.go @@ -39,29 +39,27 @@ type StateApplier struct { imageScanningReporterService *components.ImageScanningReporterServiceK8sObject applier AgentComponentApplier log logr.Logger - agentNamespace string } func NewStateApplier(apiReader client.Reader, agentComponentApplier AgentComponentApplier, k8sVersion, agentNamespace string, tlsSecretsValuesCreator components.TlsSecretsValuesCreator, log logr.Logger) *StateApplier { return &StateApplier{ - desiredConfigMap: components.NewConfigurationK8sObject(), - desiredRegistrySecret: components.NewRegistrySecretK8sObject(), + desiredConfigMap: components.NewConfigurationK8sObject(agentNamespace), + desiredRegistrySecret: components.NewRegistrySecretK8sObject(agentNamespace), desiredPriorityClass: components.NewPriorityClassK8sObject(k8sVersion), - desiredMonitorDeployment: components.NewMonitorDeploymentK8sObject(), - enforcerTlsSecret: components.NewEnforcerTlsK8sObject(tlsSecretsValuesCreator), - enforcerDeployment: components.NewEnforcerDeploymentK8sObject(), - enforcerService: components.NewEnforcerServiceK8sObject(), - enforcerValidatingWebhook: components.NewEnforcerValidatingWebhookK8sObject(k8sVersion), - enforcerMutatingWebhook: components.NewEnforcerMutatingWebhookK8sObject(k8sVersion), - stateReporterDeployment: components.NewStateReporterDeploymentK8sObject(), - resolverDeployment: components.NewResolverDeploymentK8sObject(apiReader), - resolverService: components.NewResolverServiceK8sObject(), - sensorDaemonSet: components.NewSensorDaemonSetK8sObject(), - imageScanningReporterDeployment: components.NewImageScanningReporterDeploymentK8sObject(), - imageScanningReporterService: components.NewImageScanningReporterServiceK8sObject(), + desiredMonitorDeployment: components.NewMonitorDeploymentK8sObject(agentNamespace), + enforcerTlsSecret: components.NewEnforcerTlsK8sObject(agentNamespace, tlsSecretsValuesCreator), + enforcerDeployment: components.NewEnforcerDeploymentK8sObject(agentNamespace), + enforcerService: components.NewEnforcerServiceK8sObject(agentNamespace), + enforcerValidatingWebhook: components.NewEnforcerValidatingWebhookK8sObject(agentNamespace, k8sVersion), + enforcerMutatingWebhook: components.NewEnforcerMutatingWebhookK8sObject(agentNamespace, k8sVersion), + stateReporterDeployment: components.NewStateReporterDeploymentK8sObject(agentNamespace), + resolverDeployment: components.NewResolverDeploymentK8sObject(agentNamespace, apiReader), + resolverService: components.NewResolverServiceK8sObject(agentNamespace), + sensorDaemonSet: components.NewSensorDaemonSetK8sObject(agentNamespace), + imageScanningReporterDeployment: components.NewImageScanningReporterDeploymentK8sObject(agentNamespace), + imageScanningReporterService: components.NewImageScanningReporterServiceK8sObject(agentNamespace), applier: agentComponentApplier, log: log, - agentNamespace: agentNamespace, } } @@ -71,7 +69,6 @@ func (c *StateApplier) GetPriorityClassEmptyK8sObject() client.Object { func (c *StateApplier) ApplyDesiredState(ctx context.Context, agentSpec *cbcontainersv1.CBContainersAgentSpec, registrySecret *models.RegistrySecretValues, setOwner applymentOptions.OwnerSetter) (bool, error) { applyOptions := applymentOptions.NewApplyOptions().SetOwnerSetter(setOwner) - c.setNamespace(c.agentNamespace) coreMutated, err := c.applyCoreComponents(ctx, agentSpec, registrySecret, applyOptions) if err != nil { @@ -139,24 +136,6 @@ func (c *StateApplier) ApplyDesiredState(ctx context.Context, agentSpec *cbconta return coreMutated || mutatedEnforcer || mutatedStateReporter || mutatedRuntimeResolver || mutatedComponentsDaemonSet || runtimeResolverDeleted || mutatedImageScanningReporter || imageScanningReporterDeleted || componentsDamonSetDeleted, nil } -// setNamespace sets the namespace to all the desired K8s objects which are namespaced. -func (c *StateApplier) setNamespace(namespace string) { - c.desiredConfigMap.Namespace = namespace - c.desiredRegistrySecret.Namespace = namespace - c.desiredMonitorDeployment.Namespace = namespace - c.enforcerTlsSecret.Namespace = namespace - c.enforcerDeployment.Namespace = namespace - c.enforcerService.Namespace = namespace - c.enforcerValidatingWebhook.ServiceNamespace = namespace - c.enforcerMutatingWebhook.ServiceNamespace = namespace - c.stateReporterDeployment.Namespace = namespace - c.resolverDeployment.Namespace = namespace - c.resolverService.Namespace = namespace - c.sensorDaemonSet.Namespace = namespace - c.imageScanningReporterDeployment.Namespace = namespace - c.imageScanningReporterService.Namespace = namespace -} - func (c *StateApplier) applyCoreComponents(ctx context.Context, agentSpec *cbcontainersv1.CBContainersAgentSpec, registrySecret *models.RegistrySecretValues, applyOptions *applymentOptions.ApplyOptions) (bool, error) { mutatedConfigmap, _, err := c.applier.Apply(ctx, c.desiredConfigMap, agentSpec, applyOptions) if err != nil { From acc95919450df77cfa57336518d83aba0857ae24 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Tue, 11 Jul 2023 17:08:00 +0300 Subject: [PATCH 22/22] Removed the public registry secret from all dataplane accounts since they shouldn't need it --- .../templates/dataplane_service_accounts.yaml | 12 ------------ config/rbac/dataplane/service_account.yaml | 12 ------------ 2 files changed, 24 deletions(-) diff --git a/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/dataplane_service_accounts.yaml b/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/dataplane_service_accounts.yaml index 3959f0ab..15d00ed9 100644 --- a/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/dataplane_service_accounts.yaml +++ b/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/dataplane_service_accounts.yaml @@ -1,46 +1,34 @@ apiVersion: v1 -imagePullSecrets: - - name: cbcontainers-operator-public-registry-secret kind: ServiceAccount metadata: name: cbcontainers-agent-node namespace: {{ default "cbcontainers-dataplane" .Values.operatorNamespace }} --- apiVersion: v1 -imagePullSecrets: - - name: cbcontainers-operator-public-registry-secret kind: ServiceAccount metadata: name: cbcontainers-enforcer namespace: {{ default "cbcontainers-dataplane" .Values.operatorNamespace }} --- apiVersion: v1 -imagePullSecrets: - - name: cbcontainers-operator-public-registry-secret kind: ServiceAccount metadata: name: cbcontainers-image-scanning namespace: {{ default "cbcontainers-dataplane" .Values.operatorNamespace }} --- apiVersion: v1 -imagePullSecrets: - - name: cbcontainers-operator-public-registry-secret kind: ServiceAccount metadata: name: cbcontainers-monitor namespace: {{ default "cbcontainers-dataplane" .Values.operatorNamespace }} --- apiVersion: v1 -imagePullSecrets: - - name: cbcontainers-operator-public-registry-secret kind: ServiceAccount metadata: name: cbcontainers-runtime-resolver namespace: {{ default "cbcontainers-dataplane" .Values.operatorNamespace }} --- apiVersion: v1 -imagePullSecrets: - - name: cbcontainers-operator-public-registry-secret kind: ServiceAccount metadata: name: cbcontainers-state-reporter diff --git a/config/rbac/dataplane/service_account.yaml b/config/rbac/dataplane/service_account.yaml index 926160b3..1cbbfac7 100644 --- a/config/rbac/dataplane/service_account.yaml +++ b/config/rbac/dataplane/service_account.yaml @@ -3,46 +3,34 @@ kind: ServiceAccount metadata: name: state-reporter namespace: system -imagePullSecrets: - - name: cbcontainers-operator-public-registry-secret --- apiVersion: v1 kind: ServiceAccount metadata: name: enforcer namespace: system -imagePullSecrets: - - name: cbcontainers-operator-public-registry-secret --- apiVersion: v1 kind: ServiceAccount metadata: name: monitor namespace: system -imagePullSecrets: - - name: cbcontainers-operator-public-registry-secret --- apiVersion: v1 kind: ServiceAccount metadata: name: image-scanning namespace: system -imagePullSecrets: - - name: cbcontainers-operator-public-registry-secret --- apiVersion: v1 kind: ServiceAccount metadata: name: runtime-resolver namespace: system -imagePullSecrets: - - name: cbcontainers-operator-public-registry-secret --- apiVersion: v1 kind: ServiceAccount metadata: name: agent-node namespace: system -imagePullSecrets: - - name: cbcontainers-operator-public-registry-secret