CNS-1284-Add CRI-O support to the cluster scanner

api/v1/container_runtime_types.go

@@ -12,10 +12,36 @@ const (
 	CRIOContainerRuntime       ContainerEngineType = "cri-o"
 )
 
-var SupportedK8sEngineTypes = []ContainerEngineType{ContainerdContainerRuntime, DockerContainerRuntime}
+var SupportedK8sEngineTypes = []ContainerEngineType{ContainerdContainerRuntime, DockerContainerRuntime, CRIOContainerRuntime}
 
 type K8sContainerEngineSpec struct {
 	Endpoint string `json:"endpoint,omitempty"`
-	// +kubebuilder:validation:Enum:=containerd;docker-daemon
+	// +kubebuilder:validation:Enum:=containerd;docker-daemon;cri-o
 	EngineType ContainerEngineType `json:"engineType,omitempty"`
+
+	// +kubebuilder:default:=<>
+	// CRIO holds configuration values specific to the CRI-O container engine
+	CRIO CRIOSpec `json:"CRIO,omitempty"`
+}
+
+type CRIOSpec struct {
+	// StoragePath can be used to set the path used by CRI-O to store images on each node.
+	// This path will be mounted on the cluster scanner to provide access to the node's images.
+	// If the path does not match what CRI-O uses on the nodes, then images will not be found and scanned as expected.
+	// If not specified, the default location of CRI-O is used (/var/lib/containers/storage).
+	// +kubebuilder:validation:Optional
+	StoragePath string `json:"storagePath,omitempty"`
+
+	// StorageConfigPath can be used to set the path to the storage configuration file used by CRI-O (if any).
+	// If not specified, the default location for storage is used (/etc/containers/storage.conf).
+	// The files does not need to exist.
+	// See https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md for more information
+	// +kubebuilder:validation:Optional
+	StorageConfigPath string `json:"storageConfigPath,omitempty"`
+
+	// ConfigPath can be used to set the path to CRI-O's configuration file.
+	// If not specified, the default location is used (/etc/crio/crio.conf).
+	// See https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md for more information.
+	// +kubebuilder:validation:Optional
+	ConfigPath string `json:"configPath,omitempty"`
 }

cbcontainers/state/components/sensor_daemon_set.go

@@ -28,16 +28,30 @@ const (
 
 	desiredConnectionTimeoutSeconds = 60
 
-	// k8s container runtime/contaienr engine endpoints
+	// k8s container runtime/container engine endpoints
 	containerdRuntimeEndpoint         = "/var/run/containerd/containerd.sock"
 	microk8sContainerdRuntimeEndpoint = "/var/snap/microk8s/common/run/containerd.sock"
 	k3sContainerdRuntimeEndpoint      = "/run/k3s/containerd/containerd.sock"
 	dockerRuntimeEndpoint             = "/var/run/dockershim.sock"
 	dockerSock                        = "/var/run/docker.sock"
 	crioRuntimeEndpoint               = "/var/run/crio/crio.sock"
 
+	// configuredContainerRuntimeVolumeName is used when the customer has specified a non-standard runtime endpoint in the CRD
+	// as this means we need a special volume+mount for this endpoint
 	configuredContainerRuntimeVolumeName = "configured-container-runtime-endpoint"
 
+	// CRI-O specific volumes since the socket is not enough to read image blobs from the host
+
+	crioStorageVolumeName       = "crio-storage"
+	crioStorageConfigVolumeName = "crio-storage-config"
+	crioConfigVolumeName        = "crio-config"
+
+	// Source: https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md and https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md
+
+	crioStorageDefaultPath       = "/var/lib/containers"
+	crioStorageConfigDefaultPath = "/etc/containers/storage.conf"
+	crioConfigDefaultPath        = "/etc/crio/crio.conf"
+
 	cndrCompanyCodeVarName = "CB_COMPANY_CODES"
 	cndrCompanyCodeKeyName = "companyCode"
 )
@@ -200,6 +214,10 @@ func (obj *SensorDaemonSetK8sObject) getExpectedVolumeCount(agentSpec *cbContain
 		if clusterScannerSpec.K8sContainerEngine.Endpoint != "" {
 			expectedVolumesCount += 1
 		}
+
+		// cri-o specific mounts to use the containers/storage library with the host's storage area
+		// one mount is for the image store on the host, one is for the storage.conf and one is for crio.conf => we expect 3 more
+		expectedVolumesCount += 3
 	}
 
 	if isCndrEnbaled(agentSpec.Components.Cndr) {
@@ -474,7 +492,6 @@ func (obj *SensorDaemonSetK8sObject) mutateCndrVolumesMounts(container *coreV1.C
 }
 
 func (obj *SensorDaemonSetK8sObject) mutateClusterScannerEnvVars(container *coreV1.Container, agentSpec *cbContainersV1.CBContainersAgentSpec) {
-
 	clusterScannerSpec := &agentSpec.Components.ClusterScanning.ClusterScannerAgent
 
 	customEnvs := []coreV1.EnvVar{
@@ -529,14 +546,48 @@ func (obj *SensorDaemonSetK8sObject) mutateClusterScannerVolumes(templatePodSpec
 		templatePodSpec.Volumes[routeIndex].HostPath.Path = clusterScannerSpec.K8sContainerEngine.Endpoint
 	}
 
+	// Ensure we have volumes for CRI-O
+	crioStorageIx := commonState.EnsureAndGetVolumeIndexForName(templatePodSpec, crioStorageVolumeName)
+	storagePath := clusterScannerSpec.K8sContainerEngine.CRIO.StoragePath
+	if storagePath == "" {
+		storagePath = crioStorageDefaultPath
+	}
+	if templatePodSpec.Volumes[crioStorageIx].HostPath == nil {
+		templatePodSpec.Volumes[crioStorageIx].HostPath = &coreV1.HostPathVolumeSource{}
+	}
+	templatePodSpec.Volumes[crioStorageIx].HostPath.Path = storagePath
+
+	crioStorageConfigIx := commonState.EnsureAndGetVolumeIndexForName(templatePodSpec, crioStorageConfigVolumeName)
+	storageConfigPath := clusterScannerSpec.K8sContainerEngine.CRIO.StorageConfigPath
+	if storageConfigPath == "" {
+		storageConfigPath = crioStorageConfigDefaultPath
+	}
+	if templatePodSpec.Volumes[crioStorageConfigIx].HostPath == nil {
+		templatePodSpec.Volumes[crioStorageConfigIx].HostPath = &coreV1.HostPathVolumeSource{}
+	}
+	templatePodSpec.Volumes[crioStorageConfigIx].HostPath.Path = storageConfigPath
+
+	crioConfigIx := commonState.EnsureAndGetVolumeIndexForName(templatePodSpec, crioConfigVolumeName)
+	configPath := clusterScannerSpec.K8sContainerEngine.CRIO.ConfigPath
+	if configPath == "" {
+		configPath = crioConfigDefaultPath
+	}
+	if templatePodSpec.Volumes[crioConfigIx].HostPath == nil {
+		templatePodSpec.Volumes[crioConfigIx].HostPath = &coreV1.HostPathVolumeSource{}
+	}
+	templatePodSpec.Volumes[crioConfigIx].HostPath.Path = configPath
+	// End CRI-O config
+
 	// mutate root-cas volume, for https certificates
 	commonState.MutateVolumesToIncludeRootCAsVolume(templatePodSpec)
 }
 
 func (obj *SensorDaemonSetK8sObject) mutateClusterScannerVolumesMounts(container *coreV1.Container, agentSpec *cbContainersV1.CBContainersAgentSpec) {
 	containerRuntimes := getClusterScannerContainerRuntimes(&agentSpec.Components.ClusterScanning.ClusterScannerAgent)
 
-	if container.VolumeMounts == nil || len(container.VolumeMounts) != len(containerRuntimes)+1 {
+	// We expect to see
+	// container runtimes mounts + root CA mount + 3 mounts for CRI-O
+	if container.VolumeMounts == nil || len(container.VolumeMounts) != (len(containerRuntimes)+1+3) {
 		container.VolumeMounts = make([]coreV1.VolumeMount, 0)
 	}
 
@@ -548,6 +599,15 @@ func (obj *SensorDaemonSetK8sObject) mutateClusterScannerVolumesMounts(container
 		index := commonState.EnsureAndGetVolumeMountIndexForName(container, name)
 		commonState.MutateVolumeMount(container, index, mountPath, true)
 	}
+
+	// mutate mounts for the CRI-O engine storage
+	crioStorageIx := commonState.EnsureAndGetVolumeMountIndexForName(container, crioStorageVolumeName)
+	// The storage MUST be R/W as file-based locks are used to enable concurrent access to the same storage from both CRI-O and our agent
+	commonState.MutateVolumeMount(container, crioStorageIx, crioStorageDefaultPath, false)
+	crioStorageConfigIx := commonState.EnsureAndGetVolumeMountIndexForName(container, crioStorageConfigVolumeName)
+	commonState.MutateVolumeMount(container, crioStorageConfigIx, crioStorageConfigDefaultPath, true)
+	crioConfigIx := commonState.EnsureAndGetVolumeMountIndexForName(container, crioConfigVolumeName)
+	commonState.MutateVolumeMount(container, crioConfigIx, crioConfigDefaultPath, true)
 }
 
 // The cluster scanner uses the container runtime unix socket to fetch the running containers to scan.

charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml

@@ -1845,12 +1845,44 @@ spec:
                           k8sContainerEngine:
                             default: {}
                             properties:
+                              CRIO:
+                                default: { }
+                                description: CRIO holds configuration values specific
+                                  to the CRI-O container engine
+                                properties:
+                                  configPath:
+                                    description: ConfigPath can be used to set the
+                                      path to CRI-O's configuration file. If not specified,
+                                      the default location is used (/etc/crio/crio.conf).
+                                      See https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md
+                                      for more information.
+                                    type: string
+                                  storageConfigPath:
+                                    description: StorageConfigPath can be used to
+                                      set the path to the storage configuration file
+                                      used by CRI-O (if any). If not specified, the
+                                      default location for storage is used (/etc/containers/storage.conf).
+                                      The files does not need to exist. See https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md
+                                      for more information
+                                    type: string
+                                  storagePath:
+                                    description: StoragePath can be used to set the
+                                      path used by CRI-O to store images on each node.
+                                      This path will be mounted on the cluster scanner
+                                      to provide access to the node's images. If the
+                                      path does not match what CRI-O uses on the nodes,
+                                      then images will not be found and scanned as
+                                      expected. If not specified, the default location
+                                      of CRI-O is used (/var/lib/containers/storage).
+                                    type: string
+                                type: object
                               endpoint:
                                 type: string
                               engineType:
                                 enum:
                                 - containerd
                                 - docker-daemon
+                                - cri-o
                                 type: string
                             type: object
                           labels:

config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml

@@ -3346,12 +3346,44 @@ spec:
                           k8sContainerEngine:
                             default: {}
                             properties:
+                              CRIO:
+                                default: {}
+                                description: CRIO holds configuration values specific
+                                  to the CRI-O container engine
+                                properties:
+                                  configPath:
+                                    description: ConfigPath can be used to set the
+                                      path to CRI-O's configuration file. If not specified,
+                                      the default location is used (/etc/crio/crio.conf).
+                                      See https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md
+                                      for more information.
+                                    type: string
+                                  storageConfigPath:
+                                    description: StorageConfigPath can be used to
+                                      set the path to the storage configuration file
+                                      used by CRI-O (if any). If not specified, the
+                                      default location for storage is used (/etc/containers/storage.conf).
+                                      The files does not need to exist. See https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md
+                                      for more information
+                                    type: string
+                                  storagePath:
+                                    description: StoragePath can be used to set the
+                                      path used by CRI-O to store images on each node.
+                                      This path will be mounted on the cluster scanner
+                                      to provide access to the node's images. If the
+                                      path does not match what CRI-O uses on the nodes,
+                                      then images will not be found and scanned as
+                                      expected. If not specified, the default location
+                                      of CRI-O is used (/var/lib/containers/storage).
+                                    type: string
+                                type: object
                               endpoint:
                                 type: string
                               engineType:
                                 enum:
                                 - containerd
                                 - docker-daemon
+                                - cri-o
                                 type: string
                             type: object
                           labels:

config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml

@@ -3140,12 +3140,43 @@ spec:
                           type: object
                         k8sContainerEngine:
                           properties:
+                            CRIO:
+                              description: CRIO holds configuration values specific
+                                to the CRI-O container engine
+                              properties:
+                                configPath:
+                                  description: ConfigPath can be used to set the path
+                                    to CRI-O's configuration file. If not specified,
+                                    the default location is used (/etc/crio/crio.conf).
+                                    See https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md
+                                    for more information.
+                                  type: string
+                                storageConfigPath:
+                                  description: StorageConfigPath can be used to set
+                                    the path to the storage configuration file used
+                                    by CRI-O (if any). If not specified, the default
+                                    location for storage is used (/etc/containers/storage.conf).
+                                    The files does not need to exist. See https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md
+                                    for more information
+                                  type: string
+                                storagePath:
+                                  description: StoragePath can be used to set the
+                                    path used by CRI-O to store images on each node.
+                                    This path will be mounted on the cluster scanner
+                                    to provide access to the node's images. If the
+                                    path does not match what CRI-O uses on the nodes,
+                                    then images will not be found and scanned as expected.
+                                    If not specified, the default location of CRI-O
+                                    is used (/var/lib/containers/storage).
+                                  type: string
+                              type: object
                             endpoint:
                               type: string
                             engineType:
                               enum:
                               - containerd
                               - docker-daemon
+                              - cri-o
                               type: string
                           type: object
                         labels:

controllers/cbcontainersagent_controller.go

@@ -98,7 +98,7 @@ func (r *CBContainersAgentController) Reconcile(ctx context.Context, req ctrl.Re
 	}
 
 	if err := r.setAgentDefaults(&cbContainersAgent.Spec); err != nil {
-		return ctrl.Result{}, fmt.Errorf("faild to set defaults to cluster CR: %v", err)
+		return ctrl.Result{}, fmt.Errorf("failed to set defaults to cluster CR: %v", err)
 	}
 
 	setOwner := func(controlledResource metav1.Object) error {

controllers/cluster_scanning_components_defaults.go

@@ -87,30 +87,34 @@ func (r *CBContainersAgentController) setClusterScannerAgentDefaults(clusterScan
 
 	setDefaultFileProbes(&clusterScannerAgent.Probes)
 
-	emptyK8sContainerEngineSpec := cbcontainersv1.K8sContainerEngineSpec{}
-	if clusterScannerAgent.K8sContainerEngine != emptyK8sContainerEngineSpec {
-		if err := validateClusterScannerK8sContainerEngineSpec(clusterScannerAgent.K8sContainerEngine); err != nil {
-			return err
-		}
+	if err := validateClusterScannerK8sContainerEngineSpec(clusterScannerAgent.K8sContainerEngine); err != nil {
+		return err
 	}
 
 	return nil
 }
 
 func validateClusterScannerK8sContainerEngineSpec(spec cbcontainersv1.K8sContainerEngineSpec) error {
-	if spec.Endpoint == "" {
-		return fmt.Errorf("k8s container engine endpoint must be provided if configuring k8s container engine option")
+	if spec.Endpoint == "" && spec.EngineType != "" {
+		return fmt.Errorf("k8s container engine endpoint must be provided if the k8s container engine type has been set")
 	}
 
-	if spec.EngineType == "" {
-		return fmt.Errorf("k8s container engine type must be provided if configuring k8s container engine option")
+	if spec.Endpoint != "" && spec.EngineType == "" {
+		return fmt.Errorf("k8s container engine type must be provided if the container endpoint has been set")
 	}
 
-	for _, engineType := range cbcontainersv1.SupportedK8sEngineTypes {
-		if engineType == spec.EngineType {
-			return nil
+	if spec.EngineType != "" {
+		found := false
+		for _, engineType := range cbcontainersv1.SupportedK8sEngineTypes {
+			if engineType == spec.EngineType {
+				found = true
+				break
+			}
+		}
+		if !found {
+			return fmt.Errorf("invalid engine type %v provided", spec.EngineType)
 		}
 	}
 
-	return fmt.Errorf("invalid engine type %v provided", spec.EngineType)
+	return nil
 }