From 2c2623a28be725bbee89f17efb778b8aa3c4405c Mon Sep 17 00:00:00 2001 From: BenRub Date: Wed, 30 Aug 2023 12:38:06 +0300 Subject: [PATCH 01/20] Update README.md --- charts/cbcontainers-operator/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/charts/cbcontainers-operator/README.md b/charts/cbcontainers-operator/README.md index 18e7b0a2..45366446 100644 --- a/charts/cbcontainers-operator/README.md +++ b/charts/cbcontainers-operator/README.md @@ -20,7 +20,6 @@ See [Customization](#namespace). Now, install the actual helm chart from source: ```sh -git checkout v6.0.1 # Move to the latest version of the operator cd charts/cbcontainers-operator helm install cbcontainers-operator ./cbcontainers-operator-chart ``` From 6bb0ee3ada16c885dee75f30b3ce971e60d45b10 Mon Sep 17 00:00:00 2001 From: BenRub Date: Wed, 30 Aug 2023 12:38:51 +0300 Subject: [PATCH 02/20] Update README.md --- charts/cbcontainers-agent/README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/charts/cbcontainers-agent/README.md b/charts/cbcontainers-agent/README.md index 27029489..bdc9a802 100644 --- a/charts/cbcontainers-agent/README.md +++ b/charts/cbcontainers-agent/README.md @@ -24,10 +24,8 @@ There are 8 required fields that need to be provided by the user: After setting these required fields in a `values.yaml` file you can install the chart from source ```sh -git checkout v6.0.1 # Move to the latest version of the operator cd charts/cbcontainers-agent -git checkout v6.0.0 # install the latest version of the operator -helm install cbcontainers-agent ./cbcontainers-agent-chart -f values.yaml --namespace cbcontainers-dataplane +helm install cbcontainers-agent ./cbcontainers-agent-chart ``` ## Customization From ee25a2a5dfde30a1702d42297b4e891bd0ff0689 Mon Sep 17 00:00:00 2001 From: benrub Date: Wed, 30 Aug 2023 13:25:54 +0300 Subject: [PATCH 03/20] Fix charts for main to be idetical to latest release v6.0.1 --- .../cbcontainers-operator-chart/templates/operator.yaml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml b/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml index 045ead4e..af2868e3 100644 --- a/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml +++ b/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml @@ -6404,12 +6404,7 @@ spec: type: object status: description: CBContainersAgentStatus defines the observed state of CBContainersAgent - properties: - observedGeneration: - description: ObservedGeneration is the last Custom resource generation - that was fully reconciled. - format: int64 - type: integertype: object + type: object type: object served: true storage: true From 2a8595323a54434dae4f4c679ebe1649a5aa6693 Mon Sep 17 00:00:00 2001 From: BenRub Date: Wed, 30 Aug 2023 14:11:55 +0300 Subject: [PATCH 04/20] Update README.md --- charts/cbcontainers-agent/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/cbcontainers-agent/README.md b/charts/cbcontainers-agent/README.md index bdc9a802..12809ba2 100644 --- a/charts/cbcontainers-agent/README.md +++ b/charts/cbcontainers-agent/README.md @@ -25,7 +25,7 @@ After setting these required fields in a `values.yaml` file you can install the ```sh cd charts/cbcontainers-agent -helm install cbcontainers-agent ./cbcontainers-agent-chart +helm install cbcontainers-agent ./cbcontainers-agent-chart -n cbcontainers-dataplane ``` ## Customization From 45b920bdf788a82cd4636795309fb71d781c469c Mon Sep 17 00:00:00 2001 From: benrub Date: Wed, 30 Aug 2023 16:18:59 +0300 Subject: [PATCH 05/20] Adding labels field to charts --- .../cbcontainers-agent-chart/example-values.yaml | 2 ++ .../cbcontainers-agent-chart/templates/containers-agent.yaml | 4 ++++ .../cbcontainers-agent/cbcontainers-agent-chart/values.yaml | 1 + 3 files changed, 7 insertions(+) diff --git a/charts/cbcontainers-agent/cbcontainers-agent-chart/example-values.yaml b/charts/cbcontainers-agent/cbcontainers-agent-chart/example-values.yaml index 1c33f775..3fb7cc11 100644 --- a/charts/cbcontainers-agent/cbcontainers-agent-chart/example-values.yaml +++ b/charts/cbcontainers-agent/cbcontainers-agent-chart/example-values.yaml @@ -23,6 +23,8 @@ gateways: coreEventsGatewayHost: events.gateway.com hardeningEventsGatewayHost: hardening.events.gateway.com runtimeEventsGatewayHost: runtime.events.gateway.com +labels: + my-key: my-value # components is the set of components that will be installed components: settings: diff --git a/charts/cbcontainers-agent/cbcontainers-agent-chart/templates/containers-agent.yaml b/charts/cbcontainers-agent/cbcontainers-agent-chart/templates/containers-agent.yaml index 92e6402a..4521bb6e 100644 --- a/charts/cbcontainers-agent/cbcontainers-agent-chart/templates/containers-agent.yaml +++ b/charts/cbcontainers-agent/cbcontainers-agent-chart/templates/containers-agent.yaml @@ -2,6 +2,10 @@ apiVersion: operator.containers.carbonblack.io/v1 kind: CBContainersAgent metadata: name: cbcontainers-agent + {{- with .Values.labels }} + labels: + {{- toYaml . | nindent 4 }} + {{- end }} spec: account: {{ required "orgKey is required" .Values.orgKey }} clusterName: "{{ required "clusterGroup is required" .Values.clusterGroup }}:{{ required "clusterName is required" .Values.clusterName }}" diff --git a/charts/cbcontainers-agent/cbcontainers-agent-chart/values.yaml b/charts/cbcontainers-agent/cbcontainers-agent-chart/values.yaml index 42a89ab9..1f55fb00 100644 --- a/charts/cbcontainers-agent/cbcontainers-agent-chart/values.yaml +++ b/charts/cbcontainers-agent/cbcontainers-agent-chart/values.yaml @@ -16,6 +16,7 @@ gateways: coreEventsGatewayHost: "" hardeningEventsGatewayHost: "" runtimeEventsGatewayHost: "" +labels: components: cndr: enabled: true \ No newline at end of file From 9259740ecb40b2e8fa267ede60e0dfdd0f1eff39 Mon Sep 17 00:00:00 2001 From: benrub Date: Wed, 30 Aug 2023 16:46:36 +0300 Subject: [PATCH 06/20] Adding company code secret template to charts --- .../cbcontainers-agent-chart/example-values.yaml | 6 ++++-- .../templates/_helpers.tpl | 11 +++++++++++ .../cbcontainers-company-code-secret.yaml | 15 +++++++++++++++ .../cbcontainers-agent-chart/values.yaml | 4 +++- 4 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 charts/cbcontainers-agent/cbcontainers-agent-chart/templates/cbcontainers-company-code-secret.yaml diff --git a/charts/cbcontainers-agent/cbcontainers-agent-chart/example-values.yaml b/charts/cbcontainers-agent/cbcontainers-agent-chart/example-values.yaml index 3fb7cc11..cfdef89a 100644 --- a/charts/cbcontainers-agent/cbcontainers-agent-chart/example-values.yaml +++ b/charts/cbcontainers-agent/cbcontainers-agent-chart/example-values.yaml @@ -6,7 +6,7 @@ accessToken: "" # orgKey is the ID of the organization account orgKey: "ABC123" # version is the version of the agent that will be installed -version: "3.0.0" +version: "3.0.1" # clusterGroup is the group that the cluster will belong to. clusterGroup: "default" # clusterName is the name that will be used for the cluster that the agent is installed on @@ -204,4 +204,6 @@ components: port: 7071 enabled: false cndr: - enabled: true \ No newline at end of file + enabled: true + # accessTokenSecretName is the name of the Kubernetes object of type Secret that holds the values of the Company Code + companyCodeSecretName: "my-company-code-secret-name" \ No newline at end of file diff --git a/charts/cbcontainers-agent/cbcontainers-agent-chart/templates/_helpers.tpl b/charts/cbcontainers-agent/cbcontainers-agent-chart/templates/_helpers.tpl index efda2dbf..0a1a9a67 100644 --- a/charts/cbcontainers-agent/cbcontainers-agent-chart/templates/_helpers.tpl +++ b/charts/cbcontainers-agent/cbcontainers-agent-chart/templates/_helpers.tpl @@ -8,3 +8,14 @@ {{- end -}} {{- end -}} +{{/* Get the name of the secret that contains the company code */}} +{{- define "cbcontainers-agent.company-code-name" -}} +{{- $secret := . -}} +{{- if $secret -}} +"{{- $secret -}}" +{{- else -}} +"cbcontainers-company-code" +{{- end -}} +{{- end -}} + + diff --git a/charts/cbcontainers-agent/cbcontainers-agent-chart/templates/cbcontainers-company-code-secret.yaml b/charts/cbcontainers-agent/cbcontainers-agent-chart/templates/cbcontainers-company-code-secret.yaml new file mode 100644 index 00000000..786fd88c --- /dev/null +++ b/charts/cbcontainers-agent/cbcontainers-agent-chart/templates/cbcontainers-company-code-secret.yaml @@ -0,0 +1,15 @@ +{{- /* +The Secret object will be rendered only if the accessToken value is provided. +this value is required in order for the agent components to work correctly +so not having the access token secret created here assumes that the user of the charts +created the secret in an alternative way +*/}} +{{- if .Values.companyCode -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "cbcontainers-agent.company-code-name" .Values.components.cndr.companyCodeSecretName }} + namespace: {{ default "cbcontainers-dataplane" .Values.agentNamespace }} +data: + accessToken: {{ .Values.companyCode | b64enc }} +{{- end -}} diff --git a/charts/cbcontainers-agent/cbcontainers-agent-chart/values.yaml b/charts/cbcontainers-agent/cbcontainers-agent-chart/values.yaml index 1f55fb00..8c232c21 100644 --- a/charts/cbcontainers-agent/cbcontainers-agent-chart/values.yaml +++ b/charts/cbcontainers-agent/cbcontainers-agent-chart/values.yaml @@ -1,5 +1,7 @@ -# accessToken is the API token used by the agent to communicate with the backend +# Optional: accessToken is the API token used by the agent to communicate with the backend accessToken: "" +# Optional: companyCode is the Company codes used by the agent to install the Endpoints solution +companyCode: "" # orgKey is the ID of the organization account orgKey: "" # version is the version of the agent that will be installed From f6a045c094ca658fca3b109e26cd50d7de72c932 Mon Sep 17 00:00:00 2001 From: benrub Date: Wed, 30 Aug 2023 16:52:31 +0300 Subject: [PATCH 07/20] Adding company code secret template to charts fix --- .../templates/cbcontainers-company-code-secret.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/cbcontainers-agent/cbcontainers-agent-chart/templates/cbcontainers-company-code-secret.yaml b/charts/cbcontainers-agent/cbcontainers-agent-chart/templates/cbcontainers-company-code-secret.yaml index 786fd88c..e1853758 100644 --- a/charts/cbcontainers-agent/cbcontainers-agent-chart/templates/cbcontainers-company-code-secret.yaml +++ b/charts/cbcontainers-agent/cbcontainers-agent-chart/templates/cbcontainers-company-code-secret.yaml @@ -11,5 +11,5 @@ metadata: name: {{ include "cbcontainers-agent.company-code-name" .Values.components.cndr.companyCodeSecretName }} namespace: {{ default "cbcontainers-dataplane" .Values.agentNamespace }} data: - accessToken: {{ .Values.companyCode | b64enc }} + companyCode: {{ .Values.companyCode | b64enc }} {{- end -}} From 84aa072cbda132fd7e2a32bc155dc1b67f06293a Mon Sep 17 00:00:00 2001 From: benrub Date: Wed, 30 Aug 2023 17:03:54 +0300 Subject: [PATCH 08/20] Add documentation for secrets with helm --- charts/cbcontainers-agent/README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/charts/cbcontainers-agent/README.md b/charts/cbcontainers-agent/README.md index 12809ba2..301b09d7 100644 --- a/charts/cbcontainers-agent/README.md +++ b/charts/cbcontainers-agent/README.md @@ -49,6 +49,7 @@ If the secret is pre-created before deploying the agent, then `agentNamespace` h ### Secret creation +#### Carbon Black Api Key In order for the agent components to function correctly and be able to communicate with the CBC backend an access token is required. This token is located in a secret. @@ -58,6 +59,28 @@ If that secret does not exist, the operator will not start any of the agent comp If you want to create the secret as part of the chart installation provide the `accessToken` value to the chart. +*DO NOT* store the token in your source code + +Inject this value as part of your pipeline in a secure way! + This means storing the secret as plain text in your `values.yaml` file. If you prefer to create the `Secret` yourself in an alternative and more secure way, don't set the `accessToken` value and the chart will not create the `Secret` objects. + +#### Carbon Black Company Codes +In order for the agent CNDR component to function correctly and be able to communicate with the CBC backend a company code is required. + +This code is located in a secret. +By default, the secret is named `"cbcontainers-company-code"`, but that is configurable via the `components.cndr.companyCodeSecretName` property. + +If that secret does not exist, the CNDR component will fail. + +If you want to create the secret as part of the chart installation provide the `companyCode` value to the chart. + +*DO NOT* store the code in your source code + +Inject this value as part of your pipeline in a secure way! + +This means storing the secret as plain text in your `values.yaml` file. + +If you prefer to create the `Secret` yourself in an alternative and more secure way, don't set the `companyCode` value and the chart will not create the `Secret` objects. \ No newline at end of file From 170b7581c1d552f7ee855ba624cc727784e186a2 Mon Sep 17 00:00:00 2001 From: BenRub Date: Wed, 20 Sep 2023 10:40:37 +0300 Subject: [PATCH 09/20] Update README.md --- README.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/README.md b/README.md index 5bc7536d..198e787a 100644 --- a/README.md +++ b/README.md @@ -501,3 +501,37 @@ volumes: users: - system:serviceaccount:cbcontainers-dataplane:cbcontainers-agent-node ``` + +### Uninstalling on Openshift + +```yaml +kind: SecurityContextConstraints +apiVersion: security.openshift.io/v1 +metadata: + name: scc-edr-cleaner +runAsUser: + type: RunAsAny +allowHostPID: true +allowHostPorts: false +allowHostNetwork: true +allowHostDirVolumePlugin: true +allowHostIPC: false +allowPrivilegedContainer: true +readOnlyRootFilesystem: false +seLinuxContext: + type: RunAsAny +fsGroup: + type: RunAsAny +supplementalGroups: + type: RunAsAny +volumes: +- configMap +- downwardAPI +- emptyDir +- hostPath +- persistentVolumeClaim +- projected +- secret +users: +- system:serviceaccount:cbcontainers-edr-sensor-cleaners:cbcontainers-edr-sensor-cleaner +``` From f5a90e7b3ef5df5670e4fd2ce663a5583e43f0b0 Mon Sep 17 00:00:00 2001 From: BenRub Date: Wed, 20 Sep 2023 10:54:04 +0300 Subject: [PATCH 10/20] Update README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 198e787a..95bc43b5 100644 --- a/README.md +++ b/README.md @@ -504,6 +504,9 @@ users: ### Uninstalling on Openshift +Add this SecurityContextConstraints +before running the operator uninstall command + ```yaml kind: SecurityContextConstraints apiVersion: security.openshift.io/v1 From 03862c787068b10c287374c41817c2e24bac2483 Mon Sep 17 00:00:00 2001 From: skostov Date: Tue, 26 Sep 2023 16:04:54 +0300 Subject: [PATCH 11/20] add /lib/modules and /usr/src as volume mounts. --- .../state/components/sensor_daemon_set.go | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/cbcontainers/state/components/sensor_daemon_set.go b/cbcontainers/state/components/sensor_daemon_set.go index 65ef431e..b70a48b8 100644 --- a/cbcontainers/state/components/sensor_daemon_set.go +++ b/cbcontainers/state/components/sensor_daemon_set.go @@ -77,6 +77,8 @@ var ( hostPathFile = coreV1.HostPathFile cndrHostPaths = map[string]*coreV1.HostPathVolumeSource{ "boot": {Path: "/boot", Type: &hostPathDirectory}, + "modules": {Path: "/lib/modules", Type: &hostPathDirectory}, + "src": {Path: "/usr/src", Type: &hostPathDirectory}, "cb-data-dir": {Path: "/var/opt/carbonblack", Type: &hostPathDirectoryOrCreate}, "os-release": {Path: "/etc/os-release", Type: &hostPathFile}, "root": {Path: "/", Type: &hostPathDirectory}, @@ -88,6 +90,8 @@ var ( cndrReadOnlyMounts = map[string]struct{}{ "root": {}, "boot": {}, + "modules": {}, + "src": {}, "os-release": {}, } ) @@ -121,7 +125,7 @@ func (obj *SensorDaemonSetK8sObject) MutateK8sObject(k8sObject client.Object, ag obj.initiateDaemonSet(daemonSet, agentSpec) - if commonState.IsEnabled(runtimeProtection.Enabled) || isCndrEnbaled(agentSpec.Components.Cndr) { + if commonState.IsEnabled(runtimeProtection.Enabled) || isCndrEnabled(agentSpec.Components.Cndr) { daemonSet.Spec.Template.Spec.DNSPolicy = runtimeSensorDNSPolicy daemonSet.Spec.Template.Spec.HostNetwork = runtimeSensorHostNetwork daemonSet.Spec.Template.Spec.HostPID = runtimeSensorHostPID @@ -209,14 +213,14 @@ func (obj *SensorDaemonSetK8sObject) mutateAnnotations(daemonSet *appsV1.DaemonS } } -func isCndrEnbaled(cndrSpec *cbContainersV1.CBContainersCndrSpec) bool { +func isCndrEnabled(cndrSpec *cbContainersV1.CBContainersCndrSpec) bool { return cndrSpec != nil && commonState.IsEnabled(cndrSpec.Enabled) } func (obj *SensorDaemonSetK8sObject) getExpectedVolumeCount(agentSpec *cbContainersV1.CBContainersAgentSpec) int { expectedVolumesCount := 0 - if commonState.IsEnabled(agentSpec.Components.ClusterScanning.Enabled) || isCndrEnbaled(agentSpec.Components.Cndr) { + if commonState.IsEnabled(agentSpec.Components.ClusterScanning.Enabled) || isCndrEnabled(agentSpec.Components.Cndr) { expectedVolumesCount += len(supportedContainerRuntimes) } @@ -231,7 +235,7 @@ func (obj *SensorDaemonSetK8sObject) getExpectedVolumeCount(agentSpec *cbContain expectedVolumesCount += 3 } - if isCndrEnbaled(agentSpec.Components.Cndr) { + if isCndrEnabled(agentSpec.Components.Cndr) { expectedVolumesCount += len(cndrHostPaths) } @@ -249,7 +253,7 @@ func (obj *SensorDaemonSetK8sObject) mutateVolumes(daemonSet *appsV1.DaemonSet, templatePodSpec.Volumes = make([]coreV1.Volume, 0, expectedVolumeCount) } - if commonState.IsEnabled(agentSpec.Components.ClusterScanning.Enabled) || isCndrEnbaled(agentSpec.Components.Cndr) { + if commonState.IsEnabled(agentSpec.Components.ClusterScanning.Enabled) || isCndrEnabled(agentSpec.Components.Cndr) { obj.mutateContainerRuntimesVolumes(&daemonSet.Spec.Template.Spec) } @@ -257,7 +261,7 @@ func (obj *SensorDaemonSetK8sObject) mutateVolumes(daemonSet *appsV1.DaemonSet, obj.mutateClusterScannerVolumes(&daemonSet.Spec.Template.Spec, &agentSpec.Components.ClusterScanning.ClusterScannerAgent) } - if isCndrEnbaled(agentSpec.Components.Cndr) { + if isCndrEnabled(agentSpec.Components.Cndr) { obj.mutateCndrVolumes(&daemonSet.Spec.Template.Spec, &agentSpec.Components.Cndr.Sensor) } @@ -309,7 +313,7 @@ func (obj *SensorDaemonSetK8sObject) mutateContainersList(daemonSet *appsV1.Daem desiredContainers = append(desiredContainers, clusterScannerContainer) } - if isCndrEnbaled(agentSpec.Components.Cndr) { + if isCndrEnabled(agentSpec.Components.Cndr) { cndrEnabled = true if cndrContainerLocation := obj.findContainerLocationByName(templatePodSpec.Containers, CndrContainerName); cndrContainerLocation == -1 { cndrMissing = true @@ -337,7 +341,7 @@ func (obj *SensorDaemonSetK8sObject) mutateContainersList(daemonSet *appsV1.Daem agentSpec) } - if isCndrEnbaled(agentSpec.Components.Cndr) { + if isCndrEnabled(agentSpec.Components.Cndr) { obj.mutateCndrContainer( &templatePodSpec.Containers[obj.findContainerLocationByName(templatePodSpec.Containers, CndrContainerName)], agentSpec) From e32b75381b3748f37279c3d21ea68e170e35650b Mon Sep 17 00:00:00 2001 From: benrub Date: Thu, 19 Oct 2023 12:17:06 +0300 Subject: [PATCH 12/20] Revert "add /lib/modules and /usr/src as volume mounts." This reverts commit 03862c787068b10c287374c41817c2e24bac2483. --- .../state/components/sensor_daemon_set.go | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/cbcontainers/state/components/sensor_daemon_set.go b/cbcontainers/state/components/sensor_daemon_set.go index b70a48b8..65ef431e 100644 --- a/cbcontainers/state/components/sensor_daemon_set.go +++ b/cbcontainers/state/components/sensor_daemon_set.go @@ -77,8 +77,6 @@ var ( hostPathFile = coreV1.HostPathFile cndrHostPaths = map[string]*coreV1.HostPathVolumeSource{ "boot": {Path: "/boot", Type: &hostPathDirectory}, - "modules": {Path: "/lib/modules", Type: &hostPathDirectory}, - "src": {Path: "/usr/src", Type: &hostPathDirectory}, "cb-data-dir": {Path: "/var/opt/carbonblack", Type: &hostPathDirectoryOrCreate}, "os-release": {Path: "/etc/os-release", Type: &hostPathFile}, "root": {Path: "/", Type: &hostPathDirectory}, @@ -90,8 +88,6 @@ var ( cndrReadOnlyMounts = map[string]struct{}{ "root": {}, "boot": {}, - "modules": {}, - "src": {}, "os-release": {}, } ) @@ -125,7 +121,7 @@ func (obj *SensorDaemonSetK8sObject) MutateK8sObject(k8sObject client.Object, ag obj.initiateDaemonSet(daemonSet, agentSpec) - if commonState.IsEnabled(runtimeProtection.Enabled) || isCndrEnabled(agentSpec.Components.Cndr) { + if commonState.IsEnabled(runtimeProtection.Enabled) || isCndrEnbaled(agentSpec.Components.Cndr) { daemonSet.Spec.Template.Spec.DNSPolicy = runtimeSensorDNSPolicy daemonSet.Spec.Template.Spec.HostNetwork = runtimeSensorHostNetwork daemonSet.Spec.Template.Spec.HostPID = runtimeSensorHostPID @@ -213,14 +209,14 @@ func (obj *SensorDaemonSetK8sObject) mutateAnnotations(daemonSet *appsV1.DaemonS } } -func isCndrEnabled(cndrSpec *cbContainersV1.CBContainersCndrSpec) bool { +func isCndrEnbaled(cndrSpec *cbContainersV1.CBContainersCndrSpec) bool { return cndrSpec != nil && commonState.IsEnabled(cndrSpec.Enabled) } func (obj *SensorDaemonSetK8sObject) getExpectedVolumeCount(agentSpec *cbContainersV1.CBContainersAgentSpec) int { expectedVolumesCount := 0 - if commonState.IsEnabled(agentSpec.Components.ClusterScanning.Enabled) || isCndrEnabled(agentSpec.Components.Cndr) { + if commonState.IsEnabled(agentSpec.Components.ClusterScanning.Enabled) || isCndrEnbaled(agentSpec.Components.Cndr) { expectedVolumesCount += len(supportedContainerRuntimes) } @@ -235,7 +231,7 @@ func (obj *SensorDaemonSetK8sObject) getExpectedVolumeCount(agentSpec *cbContain expectedVolumesCount += 3 } - if isCndrEnabled(agentSpec.Components.Cndr) { + if isCndrEnbaled(agentSpec.Components.Cndr) { expectedVolumesCount += len(cndrHostPaths) } @@ -253,7 +249,7 @@ func (obj *SensorDaemonSetK8sObject) mutateVolumes(daemonSet *appsV1.DaemonSet, templatePodSpec.Volumes = make([]coreV1.Volume, 0, expectedVolumeCount) } - if commonState.IsEnabled(agentSpec.Components.ClusterScanning.Enabled) || isCndrEnabled(agentSpec.Components.Cndr) { + if commonState.IsEnabled(agentSpec.Components.ClusterScanning.Enabled) || isCndrEnbaled(agentSpec.Components.Cndr) { obj.mutateContainerRuntimesVolumes(&daemonSet.Spec.Template.Spec) } @@ -261,7 +257,7 @@ func (obj *SensorDaemonSetK8sObject) mutateVolumes(daemonSet *appsV1.DaemonSet, obj.mutateClusterScannerVolumes(&daemonSet.Spec.Template.Spec, &agentSpec.Components.ClusterScanning.ClusterScannerAgent) } - if isCndrEnabled(agentSpec.Components.Cndr) { + if isCndrEnbaled(agentSpec.Components.Cndr) { obj.mutateCndrVolumes(&daemonSet.Spec.Template.Spec, &agentSpec.Components.Cndr.Sensor) } @@ -313,7 +309,7 @@ func (obj *SensorDaemonSetK8sObject) mutateContainersList(daemonSet *appsV1.Daem desiredContainers = append(desiredContainers, clusterScannerContainer) } - if isCndrEnabled(agentSpec.Components.Cndr) { + if isCndrEnbaled(agentSpec.Components.Cndr) { cndrEnabled = true if cndrContainerLocation := obj.findContainerLocationByName(templatePodSpec.Containers, CndrContainerName); cndrContainerLocation == -1 { cndrMissing = true @@ -341,7 +337,7 @@ func (obj *SensorDaemonSetK8sObject) mutateContainersList(daemonSet *appsV1.Daem agentSpec) } - if isCndrEnabled(agentSpec.Components.Cndr) { + if isCndrEnbaled(agentSpec.Components.Cndr) { obj.mutateCndrContainer( &templatePodSpec.Containers[obj.findContainerLocationByName(templatePodSpec.Containers, CndrContainerName)], agentSpec) From e88a34b570e83ee78bc9ce5bb3092839b52b5ad4 Mon Sep 17 00:00:00 2001 From: BenRub Date: Tue, 12 Sep 2023 15:33:04 +0300 Subject: [PATCH 13/20] Mount /boot to /opt/boot in CNDR container --- cbcontainers/state/components/sensor_daemon_set.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cbcontainers/state/components/sensor_daemon_set.go b/cbcontainers/state/components/sensor_daemon_set.go index 65ef431e..f67a3f89 100644 --- a/cbcontainers/state/components/sensor_daemon_set.go +++ b/cbcontainers/state/components/sensor_daemon_set.go @@ -38,6 +38,7 @@ const ( dockerSock = "/var/run/docker.sock" crioRuntimeEndpoint = "/var/run/crio/crio.sock" hostRootPath = "/var/opt/root" + bootPath = "/opt/boot" // configuredContainerRuntimeVolumeName is used when the customer has specified a non-standard runtime endpoint in the CRD // as this means we need a special volume+mount for this endpoint @@ -84,6 +85,7 @@ var ( // Optional to have a different mount volume that the host path. If not exits the host path will be used. cndrVolumeMounts = map[string]string{ "root": hostRootPath, + "boot": bootPath, } cndrReadOnlyMounts = map[string]struct{}{ "root": {}, From a4096070cc8f3da16a505dcc75121305bf4510eb Mon Sep 17 00:00:00 2001 From: benrub Date: Tue, 12 Sep 2023 15:36:58 +0300 Subject: [PATCH 14/20] Edit the version strings to v6.0.2 --- README.md | 4 ++-- charts/cbcontainers-agent/cbcontainers-agent-chart/Chart.yaml | 2 +- .../cbcontainers-operator-chart/Chart.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 95bc43b5..5ab9e6bd 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ The Carbon Black Cloud Container Operator utilizes the operator-framework to cre | Operator version | Kubernetes Sensor Component Version | Minimum Kubernetes Version | |------------------|-------------------------------------|----------------------------| -| v6.0.x | 2.10.0, 2.11.0, 2.12.0, 3.0.0 | 1.18 | +| v6.0.x | 2.10.0, 2.11.0, 2.12.0, 3.0.X | 1.18 | | v5.6.x | 2.10.0, 2.11.0, 2.12.0 | 1.16 | | v5.5.x | 2.10.0, 2.11.0 | 1.16 | @@ -27,7 +27,7 @@ Kubernetes 1.18+ is supported. ### From script: ``` -export OPERATOR_VERSION=v6.0.1 +export OPERATOR_VERSION=v6.0.2 export OPERATOR_SCRIPT_URL=https://setup.containers.carbonblack.io/$OPERATOR_VERSION/operator-apply.sh curl -s $OPERATOR_SCRIPT_URL | bash ``` diff --git a/charts/cbcontainers-agent/cbcontainers-agent-chart/Chart.yaml b/charts/cbcontainers-agent/cbcontainers-agent-chart/Chart.yaml index 921b7571..45bd4a3a 100644 --- a/charts/cbcontainers-agent/cbcontainers-agent-chart/Chart.yaml +++ b/charts/cbcontainers-agent/cbcontainers-agent-chart/Chart.yaml @@ -3,4 +3,4 @@ name: cbcontainers-agent description: A Helm chart for installing the CBContainers Agent type: application version: 2.0.0 -appVersion: "3.0.1" +appVersion: "3.0.2" diff --git a/charts/cbcontainers-operator/cbcontainers-operator-chart/Chart.yaml b/charts/cbcontainers-operator/cbcontainers-operator-chart/Chart.yaml index 407324b3..426b957f 100644 --- a/charts/cbcontainers-operator/cbcontainers-operator-chart/Chart.yaml +++ b/charts/cbcontainers-operator/cbcontainers-operator-chart/Chart.yaml @@ -3,4 +3,4 @@ name: cbcontainers-operator description: A Helm chart for installing the CBContainers operator type: application version: 2.0.0 -appVersion: v6.0.1 +appVersion: v6.0.2 From e406953d4ade8c6394769371b71fcb340f3119d8 Mon Sep 17 00:00:00 2001 From: meorio Date: Wed, 18 Oct 2023 12:03:16 +0300 Subject: [PATCH 15/20] reordering documentation (cherry picked from commit 54b7a4db460b8b9f8bb96bdff04693d0f84254d2) Signed-off-by: meorio --- README.md | 458 +----------------------------------- docs/AgentDeployment.md | 44 ++++ docs/ImageSources.md | 50 ++++ docs/LegacyCRD.md | 14 ++ docs/Main.md | 66 ++++++ docs/OpenshiftDeployment.md | 86 +++++++ docs/OperatorDeployment.md | 33 +++ docs/Prometheus.md | 52 ++++ docs/Proxy.md | 101 ++++++++ docs/Resources.md | 87 +++++++ docs/clean_cndr.sh | 58 ----- 11 files changed, 544 insertions(+), 505 deletions(-) create mode 100644 docs/AgentDeployment.md create mode 100644 docs/ImageSources.md create mode 100644 docs/LegacyCRD.md create mode 100644 docs/Main.md create mode 100644 docs/OpenshiftDeployment.md create mode 100644 docs/OperatorDeployment.md create mode 100644 docs/Prometheus.md create mode 100644 docs/Proxy.md create mode 100644 docs/Resources.md delete mode 100644 docs/clean_cndr.sh diff --git a/README.md b/README.md index 5ab9e6bd..848676cc 100644 --- a/README.md +++ b/README.md @@ -36,32 +36,22 @@ curl -s $OPERATOR_SCRIPT_URL | bash Versions list: [Releases](https://github.com/octarinesec/octarine-operator/releases) -### From Source Code -Clone the git project and deploy the operator from the source code +### OpenShift Deployment: +For OpenShift clusters, follow the OpenShift Deployment instructions: -By default, the operator utilizes CustomResourceDefinitions v1, which requires Kubernetes 1.16+. -Deploying an operator with CustomResourceDefinitions v1beta1 (deprecated in Kubernetes 1.16, removed in Kubernetes 1.22) can be done - see the relevant section below. +[OpenShift Deployment](docs/OpenshiftDeployment.md) -#### Create the operator image -``` -make docker-build docker-push IMG={IMAGE_NAME} -``` - -#### Deploy the operator resources -``` -make deploy IMG={IMAGE_NAME} -``` +* For deploying from the source code, follow the instructions in the [Operator Deployment](docs/OperatorDeployment.md) documentation * View [Developer Guide](docs/developers.md) to see how deploy the operator without using an image ## Data Plane Deployment -### 1. Apply the Carbon Black Container Api Token Secret +### 1. Apply the Carbon Black Container Api Token Secret and Company Code Secret ``` -kubectl create secret generic cbcontainers-access-token \ ---namespace cbcontainers-dataplane --from-literal=accessToken=\ -{API_Secret_Key}/{API_ID} +kubectl create secret generic cbcontainers-access-token --namespace cbcontainers-dataplane --from-literal=accessToken={API_Secret_Key}/{API_ID} +kubectl create secret generic cbcontainers-company-code --namespace cbcontainers-dataplane --from-literal=companyCode=RXXXXXXXXXXG\!XXXX ``` ### 2. Apply the Carbon Black Container Agent Custom Resource @@ -107,434 +97,8 @@ make undeploy * Notice that the above command will delete the Carbon Black Container custom resources definitions and instances. -## Reading Metrics With Prometheus - -The operator metrics are protected by kube-auth-proxy. - -You will need to grant permissions to your Prometheus server to allow it to scrape the protected metrics. - -You can create a ClusterRole and bind it with ClusterRoleBinding to the service account that your Prometheus server uses. - -If you don't have such cluster role & cluster role binding configured, you can use the following: - -Cluster Role: -```yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: -name: cbcontainers-metrics-reader -rules: -- nonResourceURLs: - - /metrics - verbs: - - get -``` - -Cluster Role binding creation: -```sh -kubectl create clusterrolebinding metrics --clusterrole=cbcontainers-metrics-reader --serviceaccount=: -``` - -## Changing components resources: -```yaml -spec: - components: - basic: - monitor: - resources: - limits: - cpu: 200m - memory: 256Mi - requests: - cpu: 30m - memory: 64Mi - enforcer: - resources: - #### DESIRED RESOURCES SPEC - for hardening enforcer container - stateReporter: - resources: - #### DESIRED RESOURCES SPEC - for hardening state reporter container - runtimeProtection: - resolver: - resources: - #### DESIRED RESOURCES SPEC - for runtime resolver container - sensor: - resources: - #### DESIRED RESOURCES SPEC - for node-agent runtime container - clusterScanning: - imageScanningReporter: - resources: - #### DESIRED RESOURCES SPEC - for image scanning reporter pod - clusterScanner: - resources: - #### DESIRED RESOURCES SPEC - for node-agent cluster-scanner container -``` -#### Cluster Scanner Component Memory -The `clusterScanning.clusterScanner` component, tries by default to scan images with size up to 1GB. -To do so, its recommended resources are: -```yaml -resources: - requests: - cpu: 100m - memory: 1Gi - limits: - cpu: 2000m - memory: 6Gi -``` - -If your images are larger than 1GB, and you want to scan them, you'll need to allocate higher memory resources in the -component's `requests.memory` & `limits.memory`, and add an environment variable `MAX_COMPRESSED_IMAGE_SIZE_MB`, to override -the max images size in MB, the scanner tries to scan. - -For example, setting the cluster scanner to be able to scan images up to 1.5 GB configuration will be: -```yaml -spec: - components: - clusterScanning: - clusterScanner: - env: - MAX_COMPRESSED_IMAGE_SIZE_MB: "1536" // 1536 MB == 1.5 GB - resources: - requests: - cpu: 100m - memory: 2Gi - limits: - cpu: 2000m - memory: 5Gi -``` - -If your nodes have low memory, and you want the cluster scanner to consume less memory, you need to reduce the -component's `requests.memory` & `limits.memory` , and override the `MAX_COMPRESSED_IMAGE_SIZE_MB`, to be less than 1GB (1024MB). - -For example, assigning lower memory resources, and set the cluster-scanner to try and scan images up to 250MB: -```yaml -spec: - components: - clusterScanning: - clusterScanner: - env: - MAX_COMPRESSED_IMAGE_SIZE_MB: "250" // 250 MB - resources: - requests: - cpu: 100m - memory: 250Mi - limits: - cpu: 2000m - memory: 1Gi -``` - -### When using Prometheus Operator - -Use the following ServiceMonitor to start scraping metrics from the CBContainers operator: -* Make sure that your Prometheus custom resource service monitor selectors match it. -```yaml -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - labels: - control-plane: operator - name: cbcontainers-operator-metrics-monitor - namespace: cbcontainers-dataplane -spec: - endpoints: - - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - path: /metrics - port: https - scheme: https - tlsConfig: - insecureSkipVerify: true - selector: - matchLabels: - control-plane: operator -``` - -## Using HTTP proxy - -Configuring the Carbon Black Cloud Container services to use HTTP proxy can be done by enabling the centralized proxy settings or by setting HTTP_PROXY, HTTPS_PROXY and NO_PROXY environment variables manually. -The centralized proxy settings apply an HTTP proxy configuration for all components, while the manual setting of environment variables allows this to be done on a per component basis. -If both HTTP proxy environment variables and centralized proxy settings are provided, the environment variables would take precedence. -The operator does not make use of the centralized proxy settings, so you have to use the environment variables for it instead. - -### Configure centralized proxy settings - -In order to configure the proxy environment variables in the Operator, use the following command to patch the Operator deployment: -```sh -kubectl set env -n cbcontainers-dataplane deployment cbcontainers-operator HTTP_PROXY="" HTTPS_PROXY="" NO_PROXY="/" -``` - -Update the `CBContainersAgent` CR with the centralized proxy settings (`kubectl edit cbcontainersagents.operator.containers.carbonblack.io cbcontainers-agent`): - -```yaml -spec: - components: - settings: - proxy: - enabled: true - httpProxy: "" - httpsProxy: "" - noProxy: "," -``` - -You can disable the centralized proxy settings without having to delete them, by setting the `enabled` key above to `false`. - -By default, the centralized proxy settings take care of determining the API server IP address(es) and the necessary proxy exclusions for the cbcontainers-dataplane namespace. -These determined values are automatically appended to the `noProxy` values from above or the specified `NO_PROXY` environment variable for a particular component. -However, if you wish to change those pre-determined values, you can specify the `noProxySuffix` key at the same level as the `noProxy` key. -It has the same format as the `noProxy` key and its values are treated in the same way as if they were pre-determined. -One can also force nothing to be appended to `noProxy` or `NO_PROXY` by setting `noProxySuffix` to an empty string. - -### Configure HTTP proxy environment variables (per component proxy settings) - -In order to configure those environment variables for the basic, Runtime and Image Scanning components, -update the `CBContainersAgent` CR using the proxy environment variables (`kubectl edit cbcontainersagents.operator.containers.carbonblack.io cbcontainers-agent`): - -```yaml -spec: - components: - basic: - enforcer: - env: - HTTP_PROXY: "" - HTTPS_PROXY: "" - NO_PROXY: "/" - stateReporter: - env: - HTTP_PROXY: "" - HTTPS_PROXY: "" - NO_PROXY: "/" - runtimeProtection: - resolver: - env: - HTTP_PROXY: "" - HTTPS_PROXY: "" - NO_PROXY: "/" - sensor: - env: - HTTP_PROXY: "" - HTTPS_PROXY: "" - NO_PROXY: "/,cbcontainers-runtime-resolver.cbcontainers-dataplane.svc.cluster.local" - clusterScanning: - clusterScanner: - env: - HTTP_PROXY: "" - HTTPS_PROXY: "" - NO_PROXY: "/,cbcontainers-image-scanning-reporter.cbcontainers-dataplane.svc.cluster.local" - imageScanningReporter: - env: - HTTP_PROXY: "" - HTTPS_PROXY: "" - NO_PROXY: "/" -``` - -It is very important to configure the NO_PROXY environment variable with the value of the Kubernetes API server IP. - -Finding the API-server IP: -```sh -kubectl -n default get service kubernetes -o=jsonpath='{..clusterIP}' -``` - -### Other proxy considerations - -When using non-transparent HTTPS proxy you will need to configure the agent to use the proxy certificate authority: -```yaml -spec: - gateways: - gatewayTLS: - rootCAsBundle: -``` -Another option will be to allow the agent communicate without verifying the certificate. this option is not recommended and exposes the agent to MITM attack. -```yaml -spec: - gateways: - gatewayTLS: - insecureSkipVerify: true -``` - -## Changing the source of the images - -By default, all the images for the operator and agent deployment are going to be pulled from Docker Hub. - -We understand that some companies might not want to pull images from Docker Hub and would prefer to mirror them into their internal repositories. - -For that reason, we allow specifying the image yourself. -To do that modify the `CBContainersAgent` resource you're applying to your cluster. +## Helm Charts Documentation +[VMware Carbon Black Cloud Container Helm Charts Documentation](charts/README.md) -Modify the following properties to specify the image for each service: - -- monitor - `spec.components.basic.monitor.image` -- enforcer - `spec.components.basic.enforcer.image` -- state-reporter - `spec.components.basic.stateReporter.image` -- runtime-resolver - `spec.components.runtimeProtection.resolver.image` -- runtime-sensor - `spec.components.runtimeProtection.sensor.image` -- image-scanning-reporter - `spec.components.clusterScanning.imageScanningReporter.image` -- cluster-scanner - `spec.components.clusterScanning.clusterScanner.image` - -The `image` object consists of 4 properties: - -- `repository` - the repository of the image, e.g. `docker.io/my-org/monitor` -- `tag` - the version tag of the image, e.g. `1.0.0`, `latest`, etc. -- `pullPolicy` - the pull policy for that image, e.g. `IfNotPresent`, `Always`, or `Never`. - See [docs](https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy). -- `pullSecrets` - the image pull secrets that are going to be used to pull the container images. - The secrets must already exist in the cluster. - See [docs](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/). - -A sample configuration can look like this: - -```yaml -spec: - monitor: - image: - repository: docker.io/my-org/monitor - tag: 1.0.0 - pullPolicy: Always - pullSecrets: - - my-pull-secret -``` - -This means that the operator will try to run the monitor service from the `docker.io/my-org/monitor:1.0.0` container image and the kubelet will be instruted to **always** pull the image, using the `my-pull-secret` secret. - -### Using a shared secret for all images - -If you want to use just one pull secret to pull all the custom images, you don't need to add it every single image configuration. -Instead you can specify it(them) under `spec.settings.imagePullSecrets`. - -The secrets you put on that list will be added to the `imagePullSecrets` list of ALL agent workloads. - -## Utilizing v1beta1 CustomResourceDefinition versions -The operator supports Kubernetes clusters from v1.13+. -The CustomResourceDefinition APIs were in beta stage in those cluster and were later promoted to GA in v1.16. They are no longer served as of v1.22 of Kubernetes. - -To maintain compatibility, this operator offers 2 sets of CustomResourceDefinitions - one under the `apiextensions/v1beta1` API and one under `apiextensons/v1`. - -By default, all operations in the repository like `deploy` or `install` work with the v1 version of the `apiextensions` API. Utilizing `v1beta1` is supported by passing the `CRD_VERSION=v1beta1` option when running make. -Note that both `apiextensions/v1` and `apiextensions/v1beta1` versions of the CRDs are generated and maintained by `make` - only commands that use the final output work with 1 version at a time. - -For example, this command will deploy the operator resources on the current cluster but utilizing the `apiextensions/v1beta1` API version for them. - -``` -make deploy CRD_VERSION=v1beta1 -``` -## Deploying on Openshift - -The operator and its agent require elevated permissions to operate properly. However, this violates the default SecurityContextConstraints on most Openshift clusters, hence the components fail to start. -This can be fixed by applying the following custom security constraint configurations on the cluster (cluster admin priveleges required). - -```yaml -kind: SecurityContextConstraints -apiVersion: security.openshift.io/v1 -metadata: - name: scc-anyuid -runAsUser: - type: MustRunAsNonRoot -allowHostPID: false -allowHostPorts: false -allowHostNetwork: false -allowHostDirVolumePlugin: false -allowHostIPC: false -allowPrivilegedContainer: false -readOnlyRootFilesystem: true -seLinuxContext: - type: RunAsAny -fsGroup: - type: RunAsAny -supplementalGroups: - type: RunAsAny -users: -- system:serviceaccount:cbcontainers-dataplane:cbcontainers-operator -- system:serviceaccount:cbcontainers-dataplane:cbcontainers-enforcer -- system:serviceaccount:cbcontainers-dataplane:cbcontainers-state-reporter -- system:serviceaccount:cbcontainers-dataplane:cbcontainers-monitor -- system:serviceaccount:cbcontainers-dataplane:cbcontainers-runtime-resolver ---- -kind: SecurityContextConstraints -apiVersion: security.openshift.io/v1 -metadata: - name: scc-image-scanning # This probably needs to be fixed in the actual deployment -runAsUser: - type: RunAsAny -allowHostPID: false -allowHostPorts: false -allowHostNetwork: false -allowHostDirVolumePlugin: false -allowHostIPC: false -allowPrivilegedContainer: false -readOnlyRootFilesystem: false -seLinuxContext: - type: RunAsAny -fsGroup: - type: RunAsAny -supplementalGroups: - type: RunAsAny -allowedCapabilities: -- 'NET_BIND_SERVICE' -users: -- system:serviceaccount:cbcontainers-dataplane:cbcontainers-image-scanning ---- -kind: SecurityContextConstraints -apiVersion: security.openshift.io/v1 -metadata: - name: scc-node-agent -runAsUser: - type: RunAsAny -allowHostPID: true -allowHostPorts: false -allowHostNetwork: true -allowHostDirVolumePlugin: true -allowHostIPC: false -allowPrivilegedContainer: true -readOnlyRootFilesystem: false -seLinuxContext: - type: RunAsAny -fsGroup: - type: RunAsAny -supplementalGroups: - type: RunAsAny -volumes: -- configMap -- downwardAPI -- emptyDir -- hostPath -- persistentVolumeClaim -- projected -- secret -users: -- system:serviceaccount:cbcontainers-dataplane:cbcontainers-agent-node -``` - -### Uninstalling on Openshift - -Add this SecurityContextConstraints -before running the operator uninstall command - -```yaml -kind: SecurityContextConstraints -apiVersion: security.openshift.io/v1 -metadata: - name: scc-edr-cleaner -runAsUser: - type: RunAsAny -allowHostPID: true -allowHostPorts: false -allowHostNetwork: true -allowHostDirVolumePlugin: true -allowHostIPC: false -allowPrivilegedContainer: true -readOnlyRootFilesystem: false -seLinuxContext: - type: RunAsAny -fsGroup: - type: RunAsAny -supplementalGroups: - type: RunAsAny -volumes: -- configMap -- downwardAPI -- emptyDir -- hostPath -- persistentVolumeClaim -- projected -- secret -users: -- system:serviceaccount:cbcontainers-edr-sensor-cleaners:cbcontainers-edr-sensor-cleaner -``` +## Full Documentation +[VMware Carbon Black Cloud Container Operator Documentation](docs/Main.md) diff --git a/docs/AgentDeployment.md b/docs/AgentDeployment.md new file mode 100644 index 00000000..3bbc229e --- /dev/null +++ b/docs/AgentDeployment.md @@ -0,0 +1,44 @@ +## Agent Deployment + +### 1. Apply the Carbon Black Container Api Token Secret + +``` +kubectl create secret generic cbcontainers-access-token \ +--namespace cbcontainers-dataplane --from-literal=accessToken=\ +{API_Secret_Key}/{API_ID} +``` +### TODO: extra secret +### 2. Apply the Carbon Black Container Agent Custom Resource + +The operator implements controllers for the Carbon Black Container custom resources definitions + +[Full Custom Resources Definitions Documentation](docs/crds.md) + +#### 2.1 Apply the Carbon Black Container Agent CR + +cbcontainersagents.operator.containers.carbonblack.io + +This is the CR you'll need to deploy in order to trigger the operator to deploy the data plane components. + +```sh +apiVersion: operator.containers.carbonblack.io/v1 +kind: CBContainersAgent +metadata: + name: cbcontainers-agent +spec: + account: {ORG_KEY} + clusterName: {CLUSTER_GROUP}:{CLUSTER_NAME} + version: {AGENT_VERSION} + gateways: + apiGateway: + host: {API_HOST} + coreEventsGateway: + host: {CORE_EVENTS_HOST} + hardeningEventsGateway: + host: {HARDENING_EVENTS_HOST} + runtimeEventsGateway: + host: {RUNTIME_EVENTS_HOST} +``` + +* notice that without applying the api token secret, the operator will return the error: + `couldn't find access token secret k8s object` diff --git a/docs/ImageSources.md b/docs/ImageSources.md new file mode 100644 index 00000000..fc305c12 --- /dev/null +++ b/docs/ImageSources.md @@ -0,0 +1,50 @@ +## Changing the source of the images + +By default, all the images for the operator and agent deployment are going to be pulled from Docker Hub. + +We understand that some companies might not want to pull images from Docker Hub and would prefer to mirror them into their internal repositories. + +For that reason, we allow specifying the image yourself. +To do that modify the `CBContainersAgent` resource you're applying to your cluster. + +Modify the following properties to specify the image for each service: + +- monitor - `spec.components.basic.monitor.image` +- enforcer - `spec.components.basic.enforcer.image` +- state-reporter - `spec.components.basic.stateReporter.image` +- runtime-resolver - `spec.components.runtimeProtection.resolver.image` +- runtime-sensor - `spec.components.runtimeProtection.sensor.image` +- image-scanning-reporter - `spec.components.clusterScanning.imageScanningReporter.image` +- cluster-scanner - `spec.components.clusterScanning.clusterScanner.image` + +The `image` object consists of 4 properties: + +- `repository` - the repository of the image, e.g. `docker.io/my-org/monitor` +- `tag` - the version tag of the image, e.g. `1.0.0`, `latest`, etc. +- `pullPolicy` - the pull policy for that image, e.g. `IfNotPresent`, `Always`, or `Never`. + See [docs](https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy). +- `pullSecrets` - the image pull secrets that are going to be used to pull the container images. + The secrets must already exist in the cluster. + See [docs](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/). + +A sample configuration can look like this: + +```yaml +spec: + monitor: + image: + repository: docker.io/my-org/monitor + tag: 1.0.0 + pullPolicy: Always + pullSecrets: + - my-pull-secret +``` + +This means that the operator will try to run the monitor service from the `docker.io/my-org/monitor:1.0.0` container image and the kubelet will be instruted to **always** pull the image, using the `my-pull-secret` secret. + +### Using a shared secret for all images + +If you want to use just one pull secret to pull all the custom images, you don't need to add it every single image configuration. +Instead you can specify it(them) under `spec.settings.imagePullSecrets`. + +The secrets you put on that list will be added to the `imagePullSecrets` list of ALL agent workloads. diff --git a/docs/LegacyCRD.md b/docs/LegacyCRD.md new file mode 100644 index 00000000..09477b7d --- /dev/null +++ b/docs/LegacyCRD.md @@ -0,0 +1,14 @@ +## Utilizing v1beta1 CustomResourceDefinition versions +The operator supports Kubernetes clusters from v1.13+. +The CustomResourceDefinition APIs were in beta stage in those cluster and were later promoted to GA in v1.16. They are no longer served as of v1.22 of Kubernetes. + +To maintain compatibility, this operator offers 2 sets of CustomResourceDefinitions - one under the `apiextensions/v1beta1` API and one under `apiextensons/v1`. + +By default, all operations in the repository like `deploy` or `install` work with the v1 version of the `apiextensions` API. Utilizing `v1beta1` is supported by passing the `CRD_VERSION=v1beta1` option when running make. +Note that both `apiextensions/v1` and `apiextensions/v1beta1` versions of the CRDs are generated and maintained by `make` - only commands that use the final output work with 1 version at a time. + +For example, this command will deploy the operator resources on the current cluster but utilizing the `apiextensions/v1beta1` API version for them. + +``` +make deploy CRD_VERSION=v1beta1 +``` \ No newline at end of file diff --git a/docs/Main.md b/docs/Main.md new file mode 100644 index 00000000..f1126349 --- /dev/null +++ b/docs/Main.md @@ -0,0 +1,66 @@ +# VMware Carbon Black Cloud Container Operator +## Overview + +The Carbon Black Cloud Container Operator runs within a Kubernetes cluster. The Container Operator is a set of controllers which deploy and manage the VMware Carbon Black Cloud Container components. + +Capabilities +* Deploy and manage the Container Essentials product bundle (including the configuration and the image scanning for Kubernetes security)! +* Automatically fetch and deploy the Carbon Black Cloud Container private image registry secret +* Automatically register the Carbon Black Cloud Container cluster +* Manage the Container Essentials validating webhook - dynamically manage the admission control webhook to avoid possible downtime +* Monitor and report agent availability to the Carbon Black console + +The Carbon Black Cloud Container Operator utilizes the operator-framework to create a GO operator, which is responsible for managing and monitoring the Cloud Container components deployment. + +## Compatibility Matrix + +| Operator version | Kubernetes Sensor Component Version | Minimum Kubernetes Version | +|------------------|-------------------------------------|----------------------------| +| v6.0.x | 2.10.0, 2.11.0, 2.12.0, 3.0.0 | 1.18 | +| v5.6.x | 2.10.0, 2.11.0, 2.12.0 | 1.16 | +| v5.5.x | 2.10.0, 2.11.0 | 1.16 | + +## Install + +First, you need to install the CBC operator on the cluster: + +[Operator Deployment](OperatorDeployment.md) + +Then you need to deploy the CBC Agent on top of the operator: + +[Agent Deployment](AgentDeployment.md) + + + +For OpenShift clusters, follow the OpenShift Deployment instructions: + +[OpenShift Deployment](OpenshiftDeployment.md) + + +## Full Uninstall + +### Uninstalling the Carbon Black Cloud Container Operator + +```sh +make undeploy +``` + +* Notice that the above command will delete the Carbon Black Container custom resources definitions and instances. + +## Documentation +1. [Setting up Prometheus access](Prometheus.md) +2. [CRD Configuration](crds.md) +3. [Resource spec Configuration](Resources.md) +4. [Using HTTP proxy](Proxy.md) +5. [Configuring image sources](ImageSources.md) +6. [RBAC Configuration](rbac.md) +7. [Using legacy v1beta1.CustomResourceDefinition](LegacyCRD.md) + +## Developers Guide +A developers guide for building and configuring the operator: + +[Developers Guide](developers.md) + +## Helm Charts Documentation +[VMware Carbon Black Cloud Container Helm Charts Documentation](../charts/README.md) + diff --git a/docs/OpenshiftDeployment.md b/docs/OpenshiftDeployment.md new file mode 100644 index 00000000..4d483cfd --- /dev/null +++ b/docs/OpenshiftDeployment.md @@ -0,0 +1,86 @@ +## Deploying on Openshift + +The operator and its agent require elevated permissions to operate properly. However, this violates the default SecurityContextConstraints on most Openshift clusters, hence the components fail to start. +This can be fixed by applying the following custom security constraint configurations on the cluster (cluster admin priveleges required). + +```yaml +kind: SecurityContextConstraints +apiVersion: security.openshift.io/v1 +metadata: + name: scc-anyuid +runAsUser: + type: MustRunAsNonRoot +allowHostPID: false +allowHostPorts: false +allowHostNetwork: false +allowHostDirVolumePlugin: false +allowHostIPC: false +allowPrivilegedContainer: false +readOnlyRootFilesystem: true +seLinuxContext: + type: RunAsAny +fsGroup: + type: RunAsAny +supplementalGroups: + type: RunAsAny +users: +- system:serviceaccount:cbcontainers-dataplane:cbcontainers-operator +- system:serviceaccount:cbcontainers-dataplane:cbcontainers-enforcer +- system:serviceaccount:cbcontainers-dataplane:cbcontainers-state-reporter +- system:serviceaccount:cbcontainers-dataplane:cbcontainers-monitor +- system:serviceaccount:cbcontainers-dataplane:cbcontainers-runtime-resolver +--- +kind: SecurityContextConstraints +apiVersion: security.openshift.io/v1 +metadata: + name: scc-image-scanning # This probably needs to be fixed in the actual deployment +runAsUser: + type: RunAsAny +allowHostPID: false +allowHostPorts: false +allowHostNetwork: false +allowHostDirVolumePlugin: false +allowHostIPC: false +allowPrivilegedContainer: false +readOnlyRootFilesystem: false +seLinuxContext: + type: RunAsAny +fsGroup: + type: RunAsAny +supplementalGroups: + type: RunAsAny +allowedCapabilities: +- 'NET_BIND_SERVICE' +users: +- system:serviceaccount:cbcontainers-dataplane:cbcontainers-image-scanning +--- +kind: SecurityContextConstraints +apiVersion: security.openshift.io/v1 +metadata: + name: scc-node-agent +runAsUser: + type: RunAsAny +allowHostPID: true +allowHostPorts: false +allowHostNetwork: true +allowHostDirVolumePlugin: true +allowHostIPC: false +allowPrivilegedContainer: true +readOnlyRootFilesystem: false +seLinuxContext: + type: RunAsAny +fsGroup: + type: RunAsAny +supplementalGroups: + type: RunAsAny +volumes: +- configMap +- downwardAPI +- emptyDir +- hostPath +- persistentVolumeClaim +- projected +- secret +users: +- system:serviceaccount:cbcontainers-dataplane:cbcontainers-agent-node +``` diff --git a/docs/OperatorDeployment.md b/docs/OperatorDeployment.md new file mode 100644 index 00000000..d606660d --- /dev/null +++ b/docs/OperatorDeployment.md @@ -0,0 +1,33 @@ +## Operator Deployment + +### Prerequisites +Kubernetes 1.18+ is supported. + +### From script: +``` +export OPERATOR_VERSION=v6.0.1 +export OPERATOR_SCRIPT_URL=https://setup.containers.carbonblack.io/$OPERATOR_VERSION/operator-apply.sh +curl -s $OPERATOR_SCRIPT_URL | bash +``` + +{OPERATOR_VERSION} is of the format "v{VERSION}" + +Versions list: [Releases](https://github.com/octarinesec/octarine-operator/releases) + +### From Source Code +Clone the git project and deploy the operator from the source code + +By default, the operator utilizes CustomResourceDefinitions v1, which requires Kubernetes 1.16+. +Deploying an operator with CustomResourceDefinitions v1beta1 (deprecated in Kubernetes 1.16, removed in Kubernetes 1.22) can be done - see the relevant section below. + +#### Create the operator image +``` +make docker-build docker-push IMG={IMAGE_NAME} +``` + +#### Deploy the operator resources +``` +make deploy IMG={IMAGE_NAME} +``` + +* View [Developer Guide](docs/developers.md) to see how deploy the operator without using an image diff --git a/docs/Prometheus.md b/docs/Prometheus.md new file mode 100644 index 00000000..f37d1b02 --- /dev/null +++ b/docs/Prometheus.md @@ -0,0 +1,52 @@ +## Reading Metrics With Prometheus + +The operator metrics are protected by kube-auth-proxy. + +You will need to grant permissions to your Prometheus server to allow it to scrape the protected metrics. + +You can create a ClusterRole and bind it with ClusterRoleBinding to the service account that your Prometheus server uses. + +If you don't have such cluster role & cluster role binding configured, you can use the following: + +Cluster Role: +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: +name: cbcontainers-metrics-reader +rules: +- nonResourceURLs: + - /metrics + verbs: + - get +``` + +Cluster Role binding creation: +```sh +kubectl create clusterrolebinding metrics --clusterrole=cbcontainers-metrics-reader --serviceaccount=: +``` + +### When using Prometheus Operator + +Use the following ServiceMonitor to start scraping metrics from the CBContainers operator: +* Make sure that your Prometheus custom resource service monitor selectors match it. +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + control-plane: operator + name: cbcontainers-operator-metrics-monitor + namespace: cbcontainers-dataplane +spec: + endpoints: + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + path: /metrics + port: https + scheme: https + tlsConfig: + insecureSkipVerify: true + selector: + matchLabels: + control-plane: operator +``` \ No newline at end of file diff --git a/docs/Proxy.md b/docs/Proxy.md new file mode 100644 index 00000000..d0e5f2df --- /dev/null +++ b/docs/Proxy.md @@ -0,0 +1,101 @@ +## Using HTTP proxy + +Configuring the Carbon Black Cloud Container services to use HTTP proxy can be done by enabling the centralized proxy settings or by setting HTTP_PROXY, HTTPS_PROXY and NO_PROXY environment variables manually. +The centralized proxy settings apply an HTTP proxy configuration for all components, while the manual setting of environment variables allows this to be done on a per component basis. +If both HTTP proxy environment variables and centralized proxy settings are provided, the environment variables would take precedence. +The operator does not make use of the centralized proxy settings, so you have to use the environment variables for it instead. + +### Configure centralized proxy settings + +In order to configure the proxy environment variables in the Operator, use the following command to patch the Operator deployment: +```sh +kubectl set env -n cbcontainers-dataplane deployment cbcontainers-operator HTTP_PROXY="" HTTPS_PROXY="" NO_PROXY="/" +``` + +Update the `CBContainersAgent` CR with the centralized proxy settings (`kubectl edit cbcontainersagents.operator.containers.carbonblack.io cbcontainers-agent`): + +```yaml +spec: + components: + settings: + proxy: + enabled: true + httpProxy: "" + httpsProxy: "" + noProxy: "," +``` + +You can disable the centralized proxy settings without having to delete them, by setting the `enabled` key above to `false`. + +By default, the centralized proxy settings take care of determining the API server IP address(es) and the necessary proxy exclusions for the cbcontainers-dataplane namespace. +These determined values are automatically appended to the `noProxy` values from above or the specified `NO_PROXY` environment variable for a particular component. +However, if you wish to change those pre-determined values, you can specify the `noProxySuffix` key at the same level as the `noProxy` key. +It has the same format as the `noProxy` key and its values are treated in the same way as if they were pre-determined. +One can also force nothing to be appended to `noProxy` or `NO_PROXY` by setting `noProxySuffix` to an empty string. + +### Configure HTTP proxy environment variables (per component proxy settings) + +In order to configure those environment variables for the basic, Runtime and Image Scanning components, +update the `CBContainersAgent` CR using the proxy environment variables (`kubectl edit cbcontainersagents.operator.containers.carbonblack.io cbcontainers-agent`): + +```yaml +spec: + components: + basic: + enforcer: + env: + HTTP_PROXY: "" + HTTPS_PROXY: "" + NO_PROXY: "/" + stateReporter: + env: + HTTP_PROXY: "" + HTTPS_PROXY: "" + NO_PROXY: "/" + runtimeProtection: + resolver: + env: + HTTP_PROXY: "" + HTTPS_PROXY: "" + NO_PROXY: "/" + sensor: + env: + HTTP_PROXY: "" + HTTPS_PROXY: "" + NO_PROXY: "/,cbcontainers-runtime-resolver.cbcontainers-dataplane.svc.cluster.local" + clusterScanning: + clusterScanner: + env: + HTTP_PROXY: "" + HTTPS_PROXY: "" + NO_PROXY: "/,cbcontainers-image-scanning-reporter.cbcontainers-dataplane.svc.cluster.local" + imageScanningReporter: + env: + HTTP_PROXY: "" + HTTPS_PROXY: "" + NO_PROXY: "/" +``` + +It is very important to configure the NO_PROXY environment variable with the value of the Kubernetes API server IP. + +Finding the API-server IP: +```sh +kubectl -n default get service kubernetes -o=jsonpath='{..clusterIP}' +``` + +### Other proxy considerations + +When using non-transparent HTTPS proxy you will need to configure the agent to use the proxy certificate authority: +```yaml +spec: + gateways: + gatewayTLS: + rootCAsBundle: +``` +Another option will be to allow the agent communicate without verifying the certificate. this option is not recommended and exposes the agent to MITM attack. +```yaml +spec: + gateways: + gatewayTLS: + insecureSkipVerify: true +``` \ No newline at end of file diff --git a/docs/Resources.md b/docs/Resources.md new file mode 100644 index 00000000..6c962840 --- /dev/null +++ b/docs/Resources.md @@ -0,0 +1,87 @@ +## Changing components resources: +```yaml +spec: + components: + basic: + monitor: + resources: + limits: + cpu: 200m + memory: 256Mi + requests: + cpu: 30m + memory: 64Mi + enforcer: + resources: + #### DESIRED RESOURCES SPEC - for hardening enforcer container + stateReporter: + resources: + #### DESIRED RESOURCES SPEC - for hardening state reporter container + runtimeProtection: + resolver: + resources: + #### DESIRED RESOURCES SPEC - for runtime resolver container + sensor: + resources: + #### DESIRED RESOURCES SPEC - for node-agent runtime container + clusterScanning: + imageScanningReporter: + resources: + #### DESIRED RESOURCES SPEC - for image scanning reporter pod + clusterScanner: + resources: + #### DESIRED RESOURCES SPEC - for node-agent cluster-scanner container +``` +#### Cluster Scanner Component Memory +The `clusterScanning.clusterScanner` component, tries by default to scan images with size up to 1GB. +To do so, its recommended resources are: +```yaml +resources: + requests: + cpu: 100m + memory: 1Gi + limits: + cpu: 2000m + memory: 6Gi +``` + +If your images are larger than 1GB, and you want to scan them, you'll need to allocate higher memory resources in the +component's `requests.memory` & `limits.memory`, and add an environment variable `MAX_COMPRESSED_IMAGE_SIZE_MB`, to override +the max images size in MB, the scanner tries to scan. + +For example, setting the cluster scanner to be able to scan images up to 1.5 GB configuration will be: +```yaml +spec: + components: + clusterScanning: + clusterScanner: + env: + MAX_COMPRESSED_IMAGE_SIZE_MB: "1536" // 1536 MB == 1.5 GB + resources: + requests: + cpu: 100m + memory: 2Gi + limits: + cpu: 2000m + memory: 5Gi +``` + +If your nodes have low memory, and you want the cluster scanner to consume less memory, you need to reduce the +component's `requests.memory` & `limits.memory` , and override the `MAX_COMPRESSED_IMAGE_SIZE_MB`, to be less than 1GB (1024MB). + +For example, assigning lower memory resources, and set the cluster-scanner to try and scan images up to 250MB: +```yaml +spec: + components: + clusterScanning: + clusterScanner: + env: + MAX_COMPRESSED_IMAGE_SIZE_MB: "250" // 250 MB + resources: + requests: + cpu: 100m + memory: 250Mi + limits: + cpu: 2000m + memory: 1Gi +``` diff --git a/docs/clean_cndr.sh b/docs/clean_cndr.sh deleted file mode 100644 index dbd1eba1..00000000 --- a/docs/clean_cndr.sh +++ /dev/null @@ -1,58 +0,0 @@ -NS="cbcontainers-edr-sensor-cleaners" - -kubectl get namespace $NS > /dev/null 2>&1 -if [ $? -eq 0 ]; then - echo "Please delete $NS namespace before running the script" - exit 1 -fi - -echo "###### Creating $NS namespace ######" -kubectl create namespace $NS - -COUNTER=0 -for node in $(kubectl get nodes --no-headers -o custom-columns=":metadata.name"); -do - echo "###### Creating cbcontainers-edr-sensor-cleaner-$COUNTER ######" - echo "apiVersion: batch/v1 -kind: Job -metadata: - name: cbcontainers-edr-sensor-cleaner-$COUNTER - namespace: $NS -spec: - template: - spec: - volumes: - - hostPath: - path: /var/opt - type: Directory - name: opt-dir - containers: - - name: edr-sensor-cleaner - image: photon:4.0 - imagePullPolicy: IfNotPresent - securityContext: - privileged: true - runAsUser: 0 - volumeMounts: - - mountPath: /var/opt - name: opt-dir - command: [/bin/sh, -c] - args: - - rm -rf /var/opt/carbonblack - restartPolicy: OnFailure - nodeName: "$node | kubectl apply -f - - COUNTER=$(expr $COUNTER + 1) -done - -COUNTER=$(expr $COUNTER - 1) - -for (( c=$COUNTER; c>=0; c-- )) -do - echo "###### Wait for job/cbcontainers-edr-sensor-cleaner-$c to finish ######" - kubectl -n $NS wait --for=condition=complete --timeout=60s job/cbcontainers-edr-sensor-cleaner-$c -done - -echo "###### Deleting $NS namespace ######" -kubectl delete namespace $NS - -echo "###### DONE ######" From e348ca2089479ede8fdac46f89ba70901b19dd7c Mon Sep 17 00:00:00 2001 From: meorio Date: Mon, 23 Oct 2023 11:32:53 +0300 Subject: [PATCH 16/20] Fixing PR remarks (cherry picked from commit 268f2dc253cde6a4322567d79c16be177cc3ea98) Signed-off-by: meorio --- README.md | 6 ++++-- docs/LegacyCRD.md | 14 -------------- docs/Main.md | 7 ++++--- docs/OpenshiftDeployment.md | 36 ++++++++++++++++++++++++++++++++++++ docs/OperatorDeployment.md | 2 +- docs/developers.md | 7 +++++++ 6 files changed, 52 insertions(+), 20 deletions(-) delete mode 100644 docs/LegacyCRD.md diff --git a/README.md b/README.md index 848676cc..9cbd239b 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Versions list: [Releases](https://github.com/octarinesec/octarine-operator/relea ### OpenShift Deployment: For OpenShift clusters, follow the OpenShift Deployment instructions: -[OpenShift Deployment](docs/OpenshiftDeployment.md) +[OpenShift Deployment and Uninstall](docs/OpenshiftDeployment.md) * For deploying from the source code, follow the instructions in the [Operator Deployment](docs/OperatorDeployment.md) documentation @@ -92,7 +92,9 @@ spec: ### Uninstalling the Carbon Black Cloud Container Operator ```sh -make undeploy +export OPERATOR_VERSION=v6.0.2 +export OPERATOR_SCRIPT_URL=https://setup.containers.carbonblack.io/$OPERATOR_VERSION/operator-apply.sh +curl -s $OPERATOR_SCRIPT_URL | bash -s -- -u ``` * Notice that the above command will delete the Carbon Black Container custom resources definitions and instances. diff --git a/docs/LegacyCRD.md b/docs/LegacyCRD.md deleted file mode 100644 index 09477b7d..00000000 --- a/docs/LegacyCRD.md +++ /dev/null @@ -1,14 +0,0 @@ -## Utilizing v1beta1 CustomResourceDefinition versions -The operator supports Kubernetes clusters from v1.13+. -The CustomResourceDefinition APIs were in beta stage in those cluster and were later promoted to GA in v1.16. They are no longer served as of v1.22 of Kubernetes. - -To maintain compatibility, this operator offers 2 sets of CustomResourceDefinitions - one under the `apiextensions/v1beta1` API and one under `apiextensons/v1`. - -By default, all operations in the repository like `deploy` or `install` work with the v1 version of the `apiextensions` API. Utilizing `v1beta1` is supported by passing the `CRD_VERSION=v1beta1` option when running make. -Note that both `apiextensions/v1` and `apiextensions/v1beta1` versions of the CRDs are generated and maintained by `make` - only commands that use the final output work with 1 version at a time. - -For example, this command will deploy the operator resources on the current cluster but utilizing the `apiextensions/v1beta1` API version for them. - -``` -make deploy CRD_VERSION=v1beta1 -``` \ No newline at end of file diff --git a/docs/Main.md b/docs/Main.md index f1126349..a3864a23 100644 --- a/docs/Main.md +++ b/docs/Main.md @@ -34,7 +34,7 @@ Then you need to deploy the CBC Agent on top of the operator: For OpenShift clusters, follow the OpenShift Deployment instructions: -[OpenShift Deployment](OpenshiftDeployment.md) +[OpenShift Deployment and Uninstall](OpenshiftDeployment.md) ## Full Uninstall @@ -42,7 +42,9 @@ For OpenShift clusters, follow the OpenShift Deployment instructions: ### Uninstalling the Carbon Black Cloud Container Operator ```sh -make undeploy +export OPERATOR_VERSION=v6.0.2 +export OPERATOR_SCRIPT_URL=https://setup.containers.carbonblack.io/$OPERATOR_VERSION/operator-apply.sh +curl -s $OPERATOR_SCRIPT_URL | bash -s -- -u ``` * Notice that the above command will delete the Carbon Black Container custom resources definitions and instances. @@ -54,7 +56,6 @@ make undeploy 4. [Using HTTP proxy](Proxy.md) 5. [Configuring image sources](ImageSources.md) 6. [RBAC Configuration](rbac.md) -7. [Using legacy v1beta1.CustomResourceDefinition](LegacyCRD.md) ## Developers Guide A developers guide for building and configuring the operator: diff --git a/docs/OpenshiftDeployment.md b/docs/OpenshiftDeployment.md index 4d483cfd..14289023 100644 --- a/docs/OpenshiftDeployment.md +++ b/docs/OpenshiftDeployment.md @@ -84,3 +84,39 @@ volumes: users: - system:serviceaccount:cbcontainers-dataplane:cbcontainers-agent-node ``` +### Uninstalling on Openshift + +Add this SecurityContextConstraints +before running the operator uninstall command + +```yaml +kind: SecurityContextConstraints +apiVersion: security.openshift.io/v1 +metadata: + name: scc-edr-cleaner +runAsUser: + type: RunAsAny +allowHostPID: true +allowHostPorts: false +allowHostNetwork: true +allowHostDirVolumePlugin: true +allowHostIPC: false +allowPrivilegedContainer: true +readOnlyRootFilesystem: false +seLinuxContext: + type: RunAsAny +fsGroup: + type: RunAsAny +supplementalGroups: + type: RunAsAny +volumes: +- configMap +- downwardAPI +- emptyDir +- hostPath +- persistentVolumeClaim +- projected +- secret +users: +- system:serviceaccount:cbcontainers-edr-sensor-cleaners:cbcontainers-edr-sensor-cleaner +``` \ No newline at end of file diff --git a/docs/OperatorDeployment.md b/docs/OperatorDeployment.md index d606660d..a1941e7d 100644 --- a/docs/OperatorDeployment.md +++ b/docs/OperatorDeployment.md @@ -5,7 +5,7 @@ Kubernetes 1.18+ is supported. ### From script: ``` -export OPERATOR_VERSION=v6.0.1 +export OPERATOR_VERSION=v6.0.2 export OPERATOR_SCRIPT_URL=https://setup.containers.carbonblack.io/$OPERATOR_VERSION/operator-apply.sh curl -s $OPERATOR_SCRIPT_URL | bash ``` diff --git a/docs/developers.md b/docs/developers.md index bcdb3dff..02957a1f 100644 --- a/docs/developers.md +++ b/docs/developers.md @@ -34,6 +34,13 @@ spec: Change {MY-ADAPTER-NAME} to your control plane adapter name. The default value is `containers` +### Uninstalling the Carbon Black Cloud Container Operator + +```sh +make undeploy +``` +* note that this does not clean-up the Carbon Black directory from the data-plane nodes + ### Changing the security context settings #### Hardening enforcer/state_reporter security context settings From 3749bd41d7059232b750de34dd6aa88f64f71dd8 Mon Sep 17 00:00:00 2001 From: benrub Date: Tue, 31 Oct 2023 14:15:06 +0200 Subject: [PATCH 17/20] Fix duplicated env setting in operator chart --- .../templates/deployment.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/deployment.yaml b/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/deployment.yaml index 51d026f4..acf78488 100644 --- a/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/deployment.yaml +++ b/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/deployment.yaml @@ -62,6 +62,10 @@ spec: - /manager image: "{{ .Values.operator.image.repository | default "cbartifactory/octarine-operator" }}:{{ .Values.operator.image.version | default .Chart.AppVersion }}" env: + - name: OPERATOR_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace {{/* If you want to enable HTTP Proxy you need to set all 3 of the values - operator.proxy.http, operator.proxy.https and operator.proxy.noProxy */}} {{- if .Values.operator.proxy -}} - name: HTTP_PROXY @@ -101,10 +105,5 @@ spec: readOnlyRootFilesystem: true runAsNonRoot: true runAsUser: 65532 - env: - - name: OPERATOR_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace serviceAccountName: cbcontainers-operator terminationGracePeriodSeconds: 10 From 1fedcc5fcded4d8216c93bd4979cbb3bc5ef239e Mon Sep 17 00:00:00 2001 From: Meori Oransky Date: Thu, 9 Nov 2023 19:17:15 +0200 Subject: [PATCH 18/20] Update the docs Main.md compatibility matrix Replaced the compatibility matrix with a link to VMware docs --- docs/Main.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/Main.md b/docs/Main.md index a3864a23..1d44401b 100644 --- a/docs/Main.md +++ b/docs/Main.md @@ -14,11 +14,8 @@ The Carbon Black Cloud Container Operator utilizes the operator-framework to cre ## Compatibility Matrix -| Operator version | Kubernetes Sensor Component Version | Minimum Kubernetes Version | -|------------------|-------------------------------------|----------------------------| -| v6.0.x | 2.10.0, 2.11.0, 2.12.0, 3.0.0 | 1.18 | -| v5.6.x | 2.10.0, 2.11.0, 2.12.0 | 1.16 | -| v5.5.x | 2.10.0, 2.11.0 | 1.16 | +The updated compatability matrix can be found here: +[Kubernetes Sensor Operator Compatability Matrix on VMware Docs](https://docs.vmware.com/en/VMware-Carbon-Black-Cloud/services/cbc-k8-sensor-oer/GUID-69F632A4-289F-44C9-9847-25E5A9493BE9.html) ## Install From a4ec106cbafe389996bfaace9dbbe7b4181591eb Mon Sep 17 00:00:00 2001 From: Meori Oransky Date: Sun, 12 Nov 2023 18:24:46 +0200 Subject: [PATCH 19/20] Update AgentDeployment.md added extra secret inscructions --- docs/AgentDeployment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/AgentDeployment.md b/docs/AgentDeployment.md index 3bbc229e..ea02ca28 100644 --- a/docs/AgentDeployment.md +++ b/docs/AgentDeployment.md @@ -6,8 +6,8 @@ kubectl create secret generic cbcontainers-access-token \ --namespace cbcontainers-dataplane --from-literal=accessToken=\ {API_Secret_Key}/{API_ID} +kubectl create secret generic cbcontainers-company-code --namespace cbcontainers-dataplane --from-literal=companyCode=RXXXXXXXXXXG\!XXXX ``` -### TODO: extra secret ### 2. Apply the Carbon Black Container Agent Custom Resource The operator implements controllers for the Carbon Black Container custom resources definitions From a03a001197ee79a7472dbce4a8cca293131f71a4 Mon Sep 17 00:00:00 2001 From: Meori Oransky Date: Sun, 12 Nov 2023 19:06:02 +0200 Subject: [PATCH 20/20] Update README.md typo fix --- charts/cbcontainers-agent/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/cbcontainers-agent/README.md b/charts/cbcontainers-agent/README.md index 301b09d7..8b313bcc 100644 --- a/charts/cbcontainers-agent/README.md +++ b/charts/cbcontainers-agent/README.md @@ -44,7 +44,7 @@ To customize that namespace, see [operator-chart](../cbcontainers-operator). The actual namespace where helm tracks the release (see [--namespace flag](https://helm.sh/docs/helm/helm_install/)) is not important to the agent chart, but the recommended approach is to also use the same namespace as the operator chart. -The `agentNamespace` value is only required if the agent chart is responsible for deploying the agent's secret as well. See [secret detection](#secret-creation) for details. +The `agentNamespace` value is only required if the agent chart is responsible for deploying the agent's secret as well. See [secret creation](#secret-creation) for details. If the secret is pre-created before deploying the agent, then `agentNamespace` has no effect. ### Secret creation @@ -83,4 +83,4 @@ Inject this value as part of your pipeline in a secure way! This means storing the secret as plain text in your `values.yaml` file. -If you prefer to create the `Secret` yourself in an alternative and more secure way, don't set the `companyCode` value and the chart will not create the `Secret` objects. \ No newline at end of file +If you prefer to create the `Secret` yourself in an alternative and more secure way, don't set the `companyCode` value and the chart will not create the `Secret` objects.