diff --git a/api/v1/container_runtime_types.go b/api/v1/container_runtime_types.go index 84d0d5db..4f854ead 100644 --- a/api/v1/container_runtime_types.go +++ b/api/v1/container_runtime_types.go @@ -12,10 +12,36 @@ const ( CRIOContainerRuntime ContainerEngineType = "cri-o" ) -var SupportedK8sEngineTypes = []ContainerEngineType{ContainerdContainerRuntime, DockerContainerRuntime} +var SupportedK8sEngineTypes = []ContainerEngineType{ContainerdContainerRuntime, DockerContainerRuntime, CRIOContainerRuntime} type K8sContainerEngineSpec struct { Endpoint string `json:"endpoint,omitempty"` - // +kubebuilder:validation:Enum:=containerd;docker-daemon + // +kubebuilder:validation:Enum:=containerd;docker-daemon;cri-o EngineType ContainerEngineType `json:"engineType,omitempty"` + + // +kubebuilder:default:=<> + // CRIO holds configuration values specific to the CRI-O container engine + CRIO CRIOSpec `json:"CRIO,omitempty"` +} + +type CRIOSpec struct { + // StoragePath can be used to set the path used by CRI-O to store images on each node. + // This path will be mounted on the cluster scanner to provide access to the node's images. + // If the path does not match what CRI-O uses on the nodes, then images will not be found and scanned as expected. + // If not specified, the default location of CRI-O is used (/var/lib/containers/storage). + // +kubebuilder:validation:Optional + StoragePath string `json:"storagePath,omitempty"` + + // StorageConfigPath can be used to set the path to the storage configuration file used by CRI-O (if any). + // If not specified, the default location for storage is used (/etc/containers/storage.conf). + // The files does not need to exist. + // See https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md for more information + // +kubebuilder:validation:Optional + StorageConfigPath string `json:"storageConfigPath,omitempty"` + + // ConfigPath can be used to set the path to CRI-O's configuration file. + // If not specified, the default location is used (/etc/crio/crio.conf). + // See https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md for more information. + // +kubebuilder:validation:Optional + ConfigPath string `json:"configPath,omitempty"` } diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 2b92a496..960b55c2 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -877,9 +877,25 @@ func (in *CBContainersStateReporterSpec) DeepCopy() *CBContainersStateReporterSp return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CRIOSpec) DeepCopyInto(out *CRIOSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CRIOSpec. +func (in *CRIOSpec) DeepCopy() *CRIOSpec { + if in == nil { + return nil + } + out := new(CRIOSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *K8sContainerEngineSpec) DeepCopyInto(out *K8sContainerEngineSpec) { *out = *in + out.CRIO = in.CRIO } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new K8sContainerEngineSpec. diff --git a/cbcontainers/state/components/sensor_daemon_set.go b/cbcontainers/state/components/sensor_daemon_set.go index fbaacad5..404daeba 100644 --- a/cbcontainers/state/components/sensor_daemon_set.go +++ b/cbcontainers/state/components/sensor_daemon_set.go @@ -28,7 +28,7 @@ const ( desiredConnectionTimeoutSeconds = 60 - // k8s container runtime/contaienr engine endpoints + // k8s container runtime/container engine endpoints containerdRuntimeEndpoint = "/var/run/containerd/containerd.sock" microk8sContainerdRuntimeEndpoint = "/var/snap/microk8s/common/run/containerd.sock" k3sContainerdRuntimeEndpoint = "/run/k3s/containerd/containerd.sock" @@ -36,8 +36,22 @@ const ( dockerSock = "/var/run/docker.sock" crioRuntimeEndpoint = "/var/run/crio/crio.sock" + // configuredContainerRuntimeVolumeName is used when the customer has specified a non-standard runtime endpoint in the CRD + // as this means we need a special volume+mount for this endpoint configuredContainerRuntimeVolumeName = "configured-container-runtime-endpoint" + // CRI-O specific volumes since the socket is not enough to read image blobs from the host + + crioStorageVolumeName = "crio-storage" + crioStorageConfigVolumeName = "crio-storage-config" + crioConfigVolumeName = "crio-config" + + // Source: https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md and https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md + + crioStorageDefaultPath = "/var/lib/containers" + crioStorageConfigDefaultPath = "/etc/containers/storage.conf" + crioConfigDefaultPath = "/etc/crio/crio.conf" + cndrCompanyCodeVarName = "CB_COMPANY_CODES" cndrCompanyCodeKeyName = "companyCode" ) @@ -192,8 +206,15 @@ func (obj *SensorDaemonSetK8sObject) getExpectedVolumeCount(agentSpec *cbContain if commonState.IsEnabled(agentSpec.Components.ClusterScanning.Enabled) { clusterScannerSpec := &agentSpec.Components.ClusterScanning.ClusterScannerAgent - containerRuntimes := getContainerRuntimes(clusterScannerSpec) - expectedVolumesCount += len(containerRuntimes) + 1 + expectedVolumesCount += len(supportedContainerRuntimes) + expectedVolumesCount += 1 // RootCA + if clusterScannerSpec.K8sContainerEngine.Endpoint != "" { + expectedVolumesCount += 1 + } + + // cri-o specific mounts to use the containers/storage library with the host's storage area + // one mount is for the image store on the host, one is for the storage.conf and one is for crio.conf => we expect 3 more + expectedVolumesCount += 3 } if isCndrEnbaled(agentSpec.Components.Cndr) { @@ -206,9 +227,10 @@ func (obj *SensorDaemonSetK8sObject) getExpectedVolumeCount(agentSpec *cbContain func (obj *SensorDaemonSetK8sObject) mutateVolumes(daemonSet *appsV1.DaemonSet, agentSpec *cbContainersV1.CBContainersAgentSpec) { templatePodSpec := &daemonSet.Spec.Template.Spec - if templatePodSpec.Volumes == nil || len(templatePodSpec.Volumes) != obj.getExpectedVolumeCount(agentSpec) { + expectedVolumeCount := obj.getExpectedVolumeCount(agentSpec) + if templatePodSpec.Volumes == nil || len(templatePodSpec.Volumes) != expectedVolumeCount { // clean cluster-scanner & cndr volumes - templatePodSpec.Volumes = make([]coreV1.Volume, 0) + templatePodSpec.Volumes = make([]coreV1.Volume, 0, expectedVolumeCount) } if commonState.IsEnabled(agentSpec.Components.ClusterScanning.Enabled) { @@ -454,7 +476,6 @@ func (obj *SensorDaemonSetK8sObject) mutateCndrVolumesMounts(container *coreV1.C } func (obj *SensorDaemonSetK8sObject) mutateClusterScannerEnvVars(container *coreV1.Container, agentSpec *cbContainersV1.CBContainersAgentSpec) { - clusterScannerSpec := &agentSpec.Components.ClusterScanning.ClusterScannerAgent customEnvs := []coreV1.EnvVar{ @@ -502,12 +523,46 @@ func (obj *SensorDaemonSetK8sObject) mutateClusterScannerVolumes(templatePodSpec } templatePodSpec.Volumes[routeIndex].HostPath.Path = path } + + // Ensure we have volumes for CRI-O + crioStorageIx := commonState.EnsureAndGetVolumeIndexForName(templatePodSpec, crioStorageVolumeName) + storagePath := clusterScannerSpec.K8sContainerEngine.CRIO.StoragePath + if storagePath == "" { + storagePath = crioStorageDefaultPath + } + if templatePodSpec.Volumes[crioStorageIx].HostPath == nil { + templatePodSpec.Volumes[crioStorageIx].HostPath = &coreV1.HostPathVolumeSource{} + } + templatePodSpec.Volumes[crioStorageIx].HostPath.Path = storagePath + + crioStorageConfigIx := commonState.EnsureAndGetVolumeIndexForName(templatePodSpec, crioStorageConfigVolumeName) + storageConfigPath := clusterScannerSpec.K8sContainerEngine.CRIO.StorageConfigPath + if storageConfigPath == "" { + storageConfigPath = crioStorageConfigDefaultPath + } + if templatePodSpec.Volumes[crioStorageConfigIx].HostPath == nil { + templatePodSpec.Volumes[crioStorageConfigIx].HostPath = &coreV1.HostPathVolumeSource{} + } + templatePodSpec.Volumes[crioStorageConfigIx].HostPath.Path = storageConfigPath + + crioConfigIx := commonState.EnsureAndGetVolumeIndexForName(templatePodSpec, crioConfigVolumeName) + configPath := clusterScannerSpec.K8sContainerEngine.CRIO.ConfigPath + if configPath == "" { + configPath = crioConfigDefaultPath + } + if templatePodSpec.Volumes[crioConfigIx].HostPath == nil { + templatePodSpec.Volumes[crioConfigIx].HostPath = &coreV1.HostPathVolumeSource{} + } + templatePodSpec.Volumes[crioConfigIx].HostPath.Path = configPath + // End CRI-O config } func (obj *SensorDaemonSetK8sObject) mutateClusterScannerVolumesMounts(container *coreV1.Container, agentSpec *cbContainersV1.CBContainersAgentSpec) { containerRuntimes := getContainerRuntimes(&agentSpec.Components.ClusterScanning.ClusterScannerAgent) - if container.VolumeMounts == nil || len(container.VolumeMounts) != len(containerRuntimes)+1 { + // We expect to see + // container runtimes mounts + root CA mount + 3 mounts for CRI-O + if container.VolumeMounts == nil || len(container.VolumeMounts) != (len(containerRuntimes)+1+3) { container.VolumeMounts = make([]coreV1.VolumeMount, 0) } @@ -519,6 +574,15 @@ func (obj *SensorDaemonSetK8sObject) mutateClusterScannerVolumesMounts(container index := commonState.EnsureAndGetVolumeMountIndexForName(container, name) commonState.MutateVolumeMount(container, index, mountPath, true) } + + // mutate mounts for the CRI-O engine storage + crioStorageIx := commonState.EnsureAndGetVolumeMountIndexForName(container, crioStorageVolumeName) + // The storage MUST be R/W as file-based locks are used to enable concurrent access to the same storage from both CRI-O and our agent + commonState.MutateVolumeMount(container, crioStorageIx, crioStorageDefaultPath, false) + crioStorageConfigIx := commonState.EnsureAndGetVolumeMountIndexForName(container, crioStorageConfigVolumeName) + commonState.MutateVolumeMount(container, crioStorageConfigIx, crioStorageConfigDefaultPath, true) + crioConfigIx := commonState.EnsureAndGetVolumeMountIndexForName(container, crioConfigVolumeName) + commonState.MutateVolumeMount(container, crioConfigIx, crioConfigDefaultPath, true) } // The cluster scanner uses the container runtime unix socket to fetch the running containers to scan. diff --git a/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml b/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml index 635220c6..98f26019 100644 --- a/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml +++ b/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml @@ -1845,12 +1845,44 @@ spec: k8sContainerEngine: default: {} properties: + CRIO: + default: { } + description: CRIO holds configuration values specific + to the CRI-O container engine + properties: + configPath: + description: ConfigPath can be used to set the + path to CRI-O's configuration file. If not specified, + the default location is used (/etc/crio/crio.conf). + See https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md + for more information. + type: string + storageConfigPath: + description: StorageConfigPath can be used to + set the path to the storage configuration file + used by CRI-O (if any). If not specified, the + default location for storage is used (/etc/containers/storage.conf). + The files does not need to exist. See https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md + for more information + type: string + storagePath: + description: StoragePath can be used to set the + path used by CRI-O to store images on each node. + This path will be mounted on the cluster scanner + to provide access to the node's images. If the + path does not match what CRI-O uses on the nodes, + then images will not be found and scanned as + expected. If not specified, the default location + of CRI-O is used (/var/lib/containers/storage). + type: string + type: object endpoint: type: string engineType: enum: - containerd - docker-daemon + - cri-o type: string type: object labels: diff --git a/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml b/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml index e1708bbd..1009057e 100644 --- a/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml +++ b/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml @@ -3346,12 +3346,44 @@ spec: k8sContainerEngine: default: {} properties: + CRIO: + default: {} + description: CRIO holds configuration values specific + to the CRI-O container engine + properties: + configPath: + description: ConfigPath can be used to set the + path to CRI-O's configuration file. If not specified, + the default location is used (/etc/crio/crio.conf). + See https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md + for more information. + type: string + storageConfigPath: + description: StorageConfigPath can be used to + set the path to the storage configuration file + used by CRI-O (if any). If not specified, the + default location for storage is used (/etc/containers/storage.conf). + The files does not need to exist. See https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md + for more information + type: string + storagePath: + description: StoragePath can be used to set the + path used by CRI-O to store images on each node. + This path will be mounted on the cluster scanner + to provide access to the node's images. If the + path does not match what CRI-O uses on the nodes, + then images will not be found and scanned as + expected. If not specified, the default location + of CRI-O is used (/var/lib/containers/storage). + type: string + type: object endpoint: type: string engineType: enum: - containerd - docker-daemon + - cri-o type: string type: object labels: diff --git a/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml b/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml index d3bd0260..233baf9c 100644 --- a/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml +++ b/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml @@ -3140,12 +3140,43 @@ spec: type: object k8sContainerEngine: properties: + CRIO: + description: CRIO holds configuration values specific + to the CRI-O container engine + properties: + configPath: + description: ConfigPath can be used to set the path + to CRI-O's configuration file. If not specified, + the default location is used (/etc/crio/crio.conf). + See https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md + for more information. + type: string + storageConfigPath: + description: StorageConfigPath can be used to set + the path to the storage configuration file used + by CRI-O (if any). If not specified, the default + location for storage is used (/etc/containers/storage.conf). + The files does not need to exist. See https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md + for more information + type: string + storagePath: + description: StoragePath can be used to set the + path used by CRI-O to store images on each node. + This path will be mounted on the cluster scanner + to provide access to the node's images. If the + path does not match what CRI-O uses on the nodes, + then images will not be found and scanned as expected. + If not specified, the default location of CRI-O + is used (/var/lib/containers/storage). + type: string + type: object endpoint: type: string engineType: enum: - containerd - docker-daemon + - cri-o type: string type: object labels: diff --git a/controllers/cbcontainersagent_controller.go b/controllers/cbcontainersagent_controller.go index 46d41548..b367844a 100644 --- a/controllers/cbcontainersagent_controller.go +++ b/controllers/cbcontainersagent_controller.go @@ -98,7 +98,7 @@ func (r *CBContainersAgentController) Reconcile(ctx context.Context, req ctrl.Re } if err := r.setAgentDefaults(&cbContainersAgent.Spec); err != nil { - return ctrl.Result{}, fmt.Errorf("faild to set defaults to cluster CR: %v", err) + return ctrl.Result{}, fmt.Errorf("failed to set defaults to cluster CR: %v", err) } setOwner := func(controlledResource metav1.Object) error { diff --git a/controllers/cluster_scanning_components_defaults.go b/controllers/cluster_scanning_components_defaults.go index b6fc1c4f..cdc0d682 100644 --- a/controllers/cluster_scanning_components_defaults.go +++ b/controllers/cluster_scanning_components_defaults.go @@ -87,30 +87,34 @@ func (r *CBContainersAgentController) setClusterScannerAgentDefaults(clusterScan setDefaultFileProbes(&clusterScannerAgent.Probes) - emptyK8sContainerEngineSpec := cbcontainersv1.K8sContainerEngineSpec{} - if clusterScannerAgent.K8sContainerEngine != emptyK8sContainerEngineSpec { - if err := validateClusterScannerK8sContainerEngineSpec(clusterScannerAgent.K8sContainerEngine); err != nil { - return err - } + if err := validateClusterScannerK8sContainerEngineSpec(clusterScannerAgent.K8sContainerEngine); err != nil { + return err } return nil } func validateClusterScannerK8sContainerEngineSpec(spec cbcontainersv1.K8sContainerEngineSpec) error { - if spec.Endpoint == "" { - return fmt.Errorf("k8s container engine endpoint must be provided if configuring k8s container engine option") + if spec.Endpoint == "" && spec.EngineType != "" { + return fmt.Errorf("k8s container engine endpoint must be provided if the k8s container engine type has been set") } - if spec.EngineType == "" { - return fmt.Errorf("k8s container engine type must be provided if configuring k8s container engine option") + if spec.Endpoint != "" && spec.EngineType == "" { + return fmt.Errorf("k8s container engine type must be provided if the container endpoint has been set") } - for _, engineType := range cbcontainersv1.SupportedK8sEngineTypes { - if engineType == spec.EngineType { - return nil + if spec.EngineType != "" { + found := false + for _, engineType := range cbcontainersv1.SupportedK8sEngineTypes { + if engineType == spec.EngineType { + found = true + break + } + } + if !found { + return fmt.Errorf("invalid engine type %v provided", spec.EngineType) } } - return fmt.Errorf("invalid engine type %v provided", spec.EngineType) + return nil } diff --git a/docs/crds.md b/docs/crds.md index 50f7a9fb..12358de8 100644 --- a/docs/crds.md +++ b/docs/crds.md @@ -58,15 +58,17 @@ This is the CR you'll need to deploy in order to trigger the operator to deploy ### Cluster Scanning Components Optional parameters -| Parameter | Description | Default | -|--------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------| -| `spec.components.clusterScanning.enabled` | Carbon Black Container flag to control Cluster Scanning components deployment | true | -| `spec.components.clusterScanning.imageScanningReporter.image.repository` | Carbon Black Container Image Scanning Reporter image repository | `cbartifactory/image-scanning-reporter` | -| `spec.components.clusterScanning.clusterScanner.image.repository` | Carbon Black Container Cluster Scanner Agent image repository | `cbartifactory/cluster-scanner` | -| `spec.components.clusterScanning.imageScanningReporter.resources` | Carbon Black Container Image Scanning Reporter resources | `{requests: {memory: "64Mi", cpu: "200m"}, limits: {memory: "1024Mi", cpu: "900m"}}` | -| `spec.components.clusterScanning.clusterScanner.resources` | Carbon Black Container Cluster Scanner resources | `{requests: {memory: "64Mi", cpu: "30m"}, limits: {memory: "1024Mi", cpu: "500m"}}` | -| `spec.components.clusterScanning.clusterScanner.k8sContainerEngine.engineType` | Carbon Black Container Cluster Scanner k8s container engine type. One of the options: `containerd`/`docker-daemon` | | -| `spec.components.clusterScanning.clusterScanner.k8sContainerEngine.endpoint` | Carbon Black Container Cluster Scanner k8s container engine endpoint path | | +| Parameter | Description | Default | +|--------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------| +| `spec.components.clusterScanning.enabled` | Carbon Black Container flag to control Cluster Scanning components deployment | true | +| `spec.components.clusterScanning.imageScanningReporter.image.repository` | Carbon Black Container Image Scanning Reporter image repository | `cbartifactory/image-scanning-reporter` | +| `spec.components.clusterScanning.clusterScanner.image.repository` | Carbon Black Container Cluster Scanner Agent image repository | `cbartifactory/cluster-scanner` | +| `spec.components.clusterScanning.imageScanningReporter.resources` | Carbon Black Container Image Scanning Reporter resources | `{requests: {memory: "64Mi", cpu: "200m"}, limits: {memory: "1024Mi", cpu: "900m"}}` | +| `spec.components.clusterScanning.clusterScanner.resources` | Carbon Black Container Cluster Scanner resources | `{requests: {memory: "64Mi", cpu: "30m"}, limits: {memory: "1024Mi", cpu: "500m"}}` | +| `spec.components.clusterScanning.clusterScanner.k8sContainerEngine.engineType` | Carbon Black Container Cluster Scanner k8s container engine type. One of the options: `containerd`/`docker-daemon`/`cri-o` | | +| `spec.components.clusterScanning.clusterScanner.k8sContainerEngine.endpoint` | Carbon Black Container Cluster Scanner k8s container engine endpoint path | | +| `spec.components.clusterScanning.clusterScanner.k8sContainerEngine.CRIO.storagePath` | Carbon Black Container Cluster Scanner override default image storage path (CRI-O only) | | +| `spec.components.clusterScanning.clusterScanner.k8sContainerEngine.CRIO.configPath` | Carbon Black Container Cluster Scanner override default image storage config path (CRI-O only) | | ### Components Common Optional parameters diff --git a/docs/developers.md b/docs/developers.md index d78afed4..f9bd6e07 100644 --- a/docs/developers.md +++ b/docs/developers.md @@ -2,7 +2,7 @@ ## Developer Guide -SKD version = 1.27.0 +SDK version = 1.27.0 ### Deploying the operator without using an image