From 43455b083d15caae0c7a3741a057d76f1a0af18e Mon Sep 17 00:00:00 2001 From: Lachezar Tsonov <80523253+ltsonov-cb@users.noreply.github.com> Date: Thu, 15 Jun 2023 17:15:01 +0300 Subject: [PATCH] CNS-1284-Add CRI-O support to the cluster scanner (#154) * Add config values for CRIO mounts * Setup CRIO mounts on the node-agent based on config values * Change CRIO config so it is optional and defaults are not set in the CR. Instead, values can be used to override the defaults from code, similar to how the rest of the Engine spec works. * Set default CRIO mounts in reconcile loop * Spellings * Fix infinite reconcile * Add new params to docs * Mount both storage.conf and crio.conf on the agent * Update helm chart, too --- api/v1/container_runtime_types.go | 30 ++++++++- api/v1/zz_generated.deepcopy.go | 16 +++++ .../state/components/sensor_daemon_set.go | 66 ++++++++++++++++++- .../templates/operator.yaml | 32 +++++++++ ...ers.carbonblack.io_cbcontainersagents.yaml | 32 +++++++++ ...ers.carbonblack.io_cbcontainersagents.yaml | 31 +++++++++ controllers/cbcontainersagent_controller.go | 2 +- .../cluster_scanning_components_defaults.go | 30 +++++---- docs/crds.md | 20 +++--- docs/developers.md | 2 +- 10 files changed, 232 insertions(+), 29 deletions(-) diff --git a/api/v1/container_runtime_types.go b/api/v1/container_runtime_types.go index 84d0d5db..4f854ead 100644 --- a/api/v1/container_runtime_types.go +++ b/api/v1/container_runtime_types.go @@ -12,10 +12,36 @@ const ( CRIOContainerRuntime ContainerEngineType = "cri-o" ) -var SupportedK8sEngineTypes = []ContainerEngineType{ContainerdContainerRuntime, DockerContainerRuntime} +var SupportedK8sEngineTypes = []ContainerEngineType{ContainerdContainerRuntime, DockerContainerRuntime, CRIOContainerRuntime} type K8sContainerEngineSpec struct { Endpoint string `json:"endpoint,omitempty"` - // +kubebuilder:validation:Enum:=containerd;docker-daemon + // +kubebuilder:validation:Enum:=containerd;docker-daemon;cri-o EngineType ContainerEngineType `json:"engineType,omitempty"` + + // +kubebuilder:default:=<> + // CRIO holds configuration values specific to the CRI-O container engine + CRIO CRIOSpec `json:"CRIO,omitempty"` +} + +type CRIOSpec struct { + // StoragePath can be used to set the path used by CRI-O to store images on each node. + // This path will be mounted on the cluster scanner to provide access to the node's images. + // If the path does not match what CRI-O uses on the nodes, then images will not be found and scanned as expected. + // If not specified, the default location of CRI-O is used (/var/lib/containers/storage). + // +kubebuilder:validation:Optional + StoragePath string `json:"storagePath,omitempty"` + + // StorageConfigPath can be used to set the path to the storage configuration file used by CRI-O (if any). + // If not specified, the default location for storage is used (/etc/containers/storage.conf). + // The files does not need to exist. + // See https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md for more information + // +kubebuilder:validation:Optional + StorageConfigPath string `json:"storageConfigPath,omitempty"` + + // ConfigPath can be used to set the path to CRI-O's configuration file. + // If not specified, the default location is used (/etc/crio/crio.conf). + // See https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md for more information. + // +kubebuilder:validation:Optional + ConfigPath string `json:"configPath,omitempty"` } diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 2b92a496..960b55c2 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -877,9 +877,25 @@ func (in *CBContainersStateReporterSpec) DeepCopy() *CBContainersStateReporterSp return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CRIOSpec) DeepCopyInto(out *CRIOSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CRIOSpec. +func (in *CRIOSpec) DeepCopy() *CRIOSpec { + if in == nil { + return nil + } + out := new(CRIOSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *K8sContainerEngineSpec) DeepCopyInto(out *K8sContainerEngineSpec) { *out = *in + out.CRIO = in.CRIO } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new K8sContainerEngineSpec. diff --git a/cbcontainers/state/components/sensor_daemon_set.go b/cbcontainers/state/components/sensor_daemon_set.go index 464ed48a..f790783c 100644 --- a/cbcontainers/state/components/sensor_daemon_set.go +++ b/cbcontainers/state/components/sensor_daemon_set.go @@ -28,7 +28,7 @@ const ( desiredConnectionTimeoutSeconds = 60 - // k8s container runtime/contaienr engine endpoints + // k8s container runtime/container engine endpoints containerdRuntimeEndpoint = "/var/run/containerd/containerd.sock" microk8sContainerdRuntimeEndpoint = "/var/snap/microk8s/common/run/containerd.sock" k3sContainerdRuntimeEndpoint = "/run/k3s/containerd/containerd.sock" @@ -36,8 +36,22 @@ const ( dockerSock = "/var/run/docker.sock" crioRuntimeEndpoint = "/var/run/crio/crio.sock" + // configuredContainerRuntimeVolumeName is used when the customer has specified a non-standard runtime endpoint in the CRD + // as this means we need a special volume+mount for this endpoint configuredContainerRuntimeVolumeName = "configured-container-runtime-endpoint" + // CRI-O specific volumes since the socket is not enough to read image blobs from the host + + crioStorageVolumeName = "crio-storage" + crioStorageConfigVolumeName = "crio-storage-config" + crioConfigVolumeName = "crio-config" + + // Source: https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md and https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md + + crioStorageDefaultPath = "/var/lib/containers" + crioStorageConfigDefaultPath = "/etc/containers/storage.conf" + crioConfigDefaultPath = "/etc/crio/crio.conf" + cndrCompanyCodeVarName = "CB_COMPANY_CODES" cndrCompanyCodeKeyName = "companyCode" ) @@ -200,6 +214,10 @@ func (obj *SensorDaemonSetK8sObject) getExpectedVolumeCount(agentSpec *cbContain if clusterScannerSpec.K8sContainerEngine.Endpoint != "" { expectedVolumesCount += 1 } + + // cri-o specific mounts to use the containers/storage library with the host's storage area + // one mount is for the image store on the host, one is for the storage.conf and one is for crio.conf => we expect 3 more + expectedVolumesCount += 3 } if isCndrEnbaled(agentSpec.Components.Cndr) { @@ -474,7 +492,6 @@ func (obj *SensorDaemonSetK8sObject) mutateCndrVolumesMounts(container *coreV1.C } func (obj *SensorDaemonSetK8sObject) mutateClusterScannerEnvVars(container *coreV1.Container, agentSpec *cbContainersV1.CBContainersAgentSpec) { - clusterScannerSpec := &agentSpec.Components.ClusterScanning.ClusterScannerAgent customEnvs := []coreV1.EnvVar{ @@ -529,6 +546,38 @@ func (obj *SensorDaemonSetK8sObject) mutateClusterScannerVolumes(templatePodSpec templatePodSpec.Volumes[routeIndex].HostPath.Path = clusterScannerSpec.K8sContainerEngine.Endpoint } + // Ensure we have volumes for CRI-O + crioStorageIx := commonState.EnsureAndGetVolumeIndexForName(templatePodSpec, crioStorageVolumeName) + storagePath := clusterScannerSpec.K8sContainerEngine.CRIO.StoragePath + if storagePath == "" { + storagePath = crioStorageDefaultPath + } + if templatePodSpec.Volumes[crioStorageIx].HostPath == nil { + templatePodSpec.Volumes[crioStorageIx].HostPath = &coreV1.HostPathVolumeSource{} + } + templatePodSpec.Volumes[crioStorageIx].HostPath.Path = storagePath + + crioStorageConfigIx := commonState.EnsureAndGetVolumeIndexForName(templatePodSpec, crioStorageConfigVolumeName) + storageConfigPath := clusterScannerSpec.K8sContainerEngine.CRIO.StorageConfigPath + if storageConfigPath == "" { + storageConfigPath = crioStorageConfigDefaultPath + } + if templatePodSpec.Volumes[crioStorageConfigIx].HostPath == nil { + templatePodSpec.Volumes[crioStorageConfigIx].HostPath = &coreV1.HostPathVolumeSource{} + } + templatePodSpec.Volumes[crioStorageConfigIx].HostPath.Path = storageConfigPath + + crioConfigIx := commonState.EnsureAndGetVolumeIndexForName(templatePodSpec, crioConfigVolumeName) + configPath := clusterScannerSpec.K8sContainerEngine.CRIO.ConfigPath + if configPath == "" { + configPath = crioConfigDefaultPath + } + if templatePodSpec.Volumes[crioConfigIx].HostPath == nil { + templatePodSpec.Volumes[crioConfigIx].HostPath = &coreV1.HostPathVolumeSource{} + } + templatePodSpec.Volumes[crioConfigIx].HostPath.Path = configPath + // End CRI-O config + // mutate root-cas volume, for https certificates commonState.MutateVolumesToIncludeRootCAsVolume(templatePodSpec) } @@ -536,7 +585,9 @@ func (obj *SensorDaemonSetK8sObject) mutateClusterScannerVolumes(templatePodSpec func (obj *SensorDaemonSetK8sObject) mutateClusterScannerVolumesMounts(container *coreV1.Container, agentSpec *cbContainersV1.CBContainersAgentSpec) { containerRuntimes := getClusterScannerContainerRuntimes(&agentSpec.Components.ClusterScanning.ClusterScannerAgent) - if container.VolumeMounts == nil || len(container.VolumeMounts) != len(containerRuntimes)+1 { + // We expect to see + // container runtimes mounts + root CA mount + 3 mounts for CRI-O + if container.VolumeMounts == nil || len(container.VolumeMounts) != (len(containerRuntimes)+1+3) { container.VolumeMounts = make([]coreV1.VolumeMount, 0) } @@ -548,6 +599,15 @@ func (obj *SensorDaemonSetK8sObject) mutateClusterScannerVolumesMounts(container index := commonState.EnsureAndGetVolumeMountIndexForName(container, name) commonState.MutateVolumeMount(container, index, mountPath, true) } + + // mutate mounts for the CRI-O engine storage + crioStorageIx := commonState.EnsureAndGetVolumeMountIndexForName(container, crioStorageVolumeName) + // The storage MUST be R/W as file-based locks are used to enable concurrent access to the same storage from both CRI-O and our agent + commonState.MutateVolumeMount(container, crioStorageIx, crioStorageDefaultPath, false) + crioStorageConfigIx := commonState.EnsureAndGetVolumeMountIndexForName(container, crioStorageConfigVolumeName) + commonState.MutateVolumeMount(container, crioStorageConfigIx, crioStorageConfigDefaultPath, true) + crioConfigIx := commonState.EnsureAndGetVolumeMountIndexForName(container, crioConfigVolumeName) + commonState.MutateVolumeMount(container, crioConfigIx, crioConfigDefaultPath, true) } // The cluster scanner uses the container runtime unix socket to fetch the running containers to scan. diff --git a/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml b/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml index 635220c6..98f26019 100644 --- a/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml +++ b/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml @@ -1845,12 +1845,44 @@ spec: k8sContainerEngine: default: {} properties: + CRIO: + default: { } + description: CRIO holds configuration values specific + to the CRI-O container engine + properties: + configPath: + description: ConfigPath can be used to set the + path to CRI-O's configuration file. If not specified, + the default location is used (/etc/crio/crio.conf). + See https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md + for more information. + type: string + storageConfigPath: + description: StorageConfigPath can be used to + set the path to the storage configuration file + used by CRI-O (if any). If not specified, the + default location for storage is used (/etc/containers/storage.conf). + The files does not need to exist. See https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md + for more information + type: string + storagePath: + description: StoragePath can be used to set the + path used by CRI-O to store images on each node. + This path will be mounted on the cluster scanner + to provide access to the node's images. If the + path does not match what CRI-O uses on the nodes, + then images will not be found and scanned as + expected. If not specified, the default location + of CRI-O is used (/var/lib/containers/storage). + type: string + type: object endpoint: type: string engineType: enum: - containerd - docker-daemon + - cri-o type: string type: object labels: diff --git a/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml b/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml index e1708bbd..1009057e 100644 --- a/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml +++ b/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml @@ -3346,12 +3346,44 @@ spec: k8sContainerEngine: default: {} properties: + CRIO: + default: {} + description: CRIO holds configuration values specific + to the CRI-O container engine + properties: + configPath: + description: ConfigPath can be used to set the + path to CRI-O's configuration file. If not specified, + the default location is used (/etc/crio/crio.conf). + See https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md + for more information. + type: string + storageConfigPath: + description: StorageConfigPath can be used to + set the path to the storage configuration file + used by CRI-O (if any). If not specified, the + default location for storage is used (/etc/containers/storage.conf). + The files does not need to exist. See https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md + for more information + type: string + storagePath: + description: StoragePath can be used to set the + path used by CRI-O to store images on each node. + This path will be mounted on the cluster scanner + to provide access to the node's images. If the + path does not match what CRI-O uses on the nodes, + then images will not be found and scanned as + expected. If not specified, the default location + of CRI-O is used (/var/lib/containers/storage). + type: string + type: object endpoint: type: string engineType: enum: - containerd - docker-daemon + - cri-o type: string type: object labels: diff --git a/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml b/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml index d3bd0260..233baf9c 100644 --- a/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml +++ b/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml @@ -3140,12 +3140,43 @@ spec: type: object k8sContainerEngine: properties: + CRIO: + description: CRIO holds configuration values specific + to the CRI-O container engine + properties: + configPath: + description: ConfigPath can be used to set the path + to CRI-O's configuration file. If not specified, + the default location is used (/etc/crio/crio.conf). + See https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md + for more information. + type: string + storageConfigPath: + description: StorageConfigPath can be used to set + the path to the storage configuration file used + by CRI-O (if any). If not specified, the default + location for storage is used (/etc/containers/storage.conf). + The files does not need to exist. See https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md + for more information + type: string + storagePath: + description: StoragePath can be used to set the + path used by CRI-O to store images on each node. + This path will be mounted on the cluster scanner + to provide access to the node's images. If the + path does not match what CRI-O uses on the nodes, + then images will not be found and scanned as expected. + If not specified, the default location of CRI-O + is used (/var/lib/containers/storage). + type: string + type: object endpoint: type: string engineType: enum: - containerd - docker-daemon + - cri-o type: string type: object labels: diff --git a/controllers/cbcontainersagent_controller.go b/controllers/cbcontainersagent_controller.go index 46d41548..b367844a 100644 --- a/controllers/cbcontainersagent_controller.go +++ b/controllers/cbcontainersagent_controller.go @@ -98,7 +98,7 @@ func (r *CBContainersAgentController) Reconcile(ctx context.Context, req ctrl.Re } if err := r.setAgentDefaults(&cbContainersAgent.Spec); err != nil { - return ctrl.Result{}, fmt.Errorf("faild to set defaults to cluster CR: %v", err) + return ctrl.Result{}, fmt.Errorf("failed to set defaults to cluster CR: %v", err) } setOwner := func(controlledResource metav1.Object) error { diff --git a/controllers/cluster_scanning_components_defaults.go b/controllers/cluster_scanning_components_defaults.go index b6fc1c4f..cdc0d682 100644 --- a/controllers/cluster_scanning_components_defaults.go +++ b/controllers/cluster_scanning_components_defaults.go @@ -87,30 +87,34 @@ func (r *CBContainersAgentController) setClusterScannerAgentDefaults(clusterScan setDefaultFileProbes(&clusterScannerAgent.Probes) - emptyK8sContainerEngineSpec := cbcontainersv1.K8sContainerEngineSpec{} - if clusterScannerAgent.K8sContainerEngine != emptyK8sContainerEngineSpec { - if err := validateClusterScannerK8sContainerEngineSpec(clusterScannerAgent.K8sContainerEngine); err != nil { - return err - } + if err := validateClusterScannerK8sContainerEngineSpec(clusterScannerAgent.K8sContainerEngine); err != nil { + return err } return nil } func validateClusterScannerK8sContainerEngineSpec(spec cbcontainersv1.K8sContainerEngineSpec) error { - if spec.Endpoint == "" { - return fmt.Errorf("k8s container engine endpoint must be provided if configuring k8s container engine option") + if spec.Endpoint == "" && spec.EngineType != "" { + return fmt.Errorf("k8s container engine endpoint must be provided if the k8s container engine type has been set") } - if spec.EngineType == "" { - return fmt.Errorf("k8s container engine type must be provided if configuring k8s container engine option") + if spec.Endpoint != "" && spec.EngineType == "" { + return fmt.Errorf("k8s container engine type must be provided if the container endpoint has been set") } - for _, engineType := range cbcontainersv1.SupportedK8sEngineTypes { - if engineType == spec.EngineType { - return nil + if spec.EngineType != "" { + found := false + for _, engineType := range cbcontainersv1.SupportedK8sEngineTypes { + if engineType == spec.EngineType { + found = true + break + } + } + if !found { + return fmt.Errorf("invalid engine type %v provided", spec.EngineType) } } - return fmt.Errorf("invalid engine type %v provided", spec.EngineType) + return nil } diff --git a/docs/crds.md b/docs/crds.md index 50f7a9fb..12358de8 100644 --- a/docs/crds.md +++ b/docs/crds.md @@ -58,15 +58,17 @@ This is the CR you'll need to deploy in order to trigger the operator to deploy ### Cluster Scanning Components Optional parameters -| Parameter | Description | Default | -|--------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------| -| `spec.components.clusterScanning.enabled` | Carbon Black Container flag to control Cluster Scanning components deployment | true | -| `spec.components.clusterScanning.imageScanningReporter.image.repository` | Carbon Black Container Image Scanning Reporter image repository | `cbartifactory/image-scanning-reporter` | -| `spec.components.clusterScanning.clusterScanner.image.repository` | Carbon Black Container Cluster Scanner Agent image repository | `cbartifactory/cluster-scanner` | -| `spec.components.clusterScanning.imageScanningReporter.resources` | Carbon Black Container Image Scanning Reporter resources | `{requests: {memory: "64Mi", cpu: "200m"}, limits: {memory: "1024Mi", cpu: "900m"}}` | -| `spec.components.clusterScanning.clusterScanner.resources` | Carbon Black Container Cluster Scanner resources | `{requests: {memory: "64Mi", cpu: "30m"}, limits: {memory: "1024Mi", cpu: "500m"}}` | -| `spec.components.clusterScanning.clusterScanner.k8sContainerEngine.engineType` | Carbon Black Container Cluster Scanner k8s container engine type. One of the options: `containerd`/`docker-daemon` | | -| `spec.components.clusterScanning.clusterScanner.k8sContainerEngine.endpoint` | Carbon Black Container Cluster Scanner k8s container engine endpoint path | | +| Parameter | Description | Default | +|--------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------| +| `spec.components.clusterScanning.enabled` | Carbon Black Container flag to control Cluster Scanning components deployment | true | +| `spec.components.clusterScanning.imageScanningReporter.image.repository` | Carbon Black Container Image Scanning Reporter image repository | `cbartifactory/image-scanning-reporter` | +| `spec.components.clusterScanning.clusterScanner.image.repository` | Carbon Black Container Cluster Scanner Agent image repository | `cbartifactory/cluster-scanner` | +| `spec.components.clusterScanning.imageScanningReporter.resources` | Carbon Black Container Image Scanning Reporter resources | `{requests: {memory: "64Mi", cpu: "200m"}, limits: {memory: "1024Mi", cpu: "900m"}}` | +| `spec.components.clusterScanning.clusterScanner.resources` | Carbon Black Container Cluster Scanner resources | `{requests: {memory: "64Mi", cpu: "30m"}, limits: {memory: "1024Mi", cpu: "500m"}}` | +| `spec.components.clusterScanning.clusterScanner.k8sContainerEngine.engineType` | Carbon Black Container Cluster Scanner k8s container engine type. One of the options: `containerd`/`docker-daemon`/`cri-o` | | +| `spec.components.clusterScanning.clusterScanner.k8sContainerEngine.endpoint` | Carbon Black Container Cluster Scanner k8s container engine endpoint path | | +| `spec.components.clusterScanning.clusterScanner.k8sContainerEngine.CRIO.storagePath` | Carbon Black Container Cluster Scanner override default image storage path (CRI-O only) | | +| `spec.components.clusterScanning.clusterScanner.k8sContainerEngine.CRIO.configPath` | Carbon Black Container Cluster Scanner override default image storage config path (CRI-O only) | | ### Components Common Optional parameters diff --git a/docs/developers.md b/docs/developers.md index d78afed4..f9bd6e07 100644 --- a/docs/developers.md +++ b/docs/developers.md @@ -2,7 +2,7 @@ ## Developer Guide -SKD version = 1.27.0 +SDK version = 1.27.0 ### Deploying the operator without using an image