CNS-1284-Add CRI-O support to the cluster scanner (#154)

* Add config values for CRIO mounts

* Setup CRIO mounts on the node-agent based on config values

* Change CRIO config so it is optional and defaults are not set in the CR. Instead, values can be used to override the defaults from code, similar to how the rest of the Engine spec works.

* Set default CRIO mounts in reconcile loop

* Spellings

* Fix infinite reconcile

* Add new params to docs

* Mount both storage.conf and crio.conf on the agent

* Update helm chart, too

api/v1/container_runtime_types.go

@@ -12,10 +12,36 @@ const (
 	CRIOContainerRuntime       ContainerEngineType = "cri-o"
 )
 
-var SupportedK8sEngineTypes = []ContainerEngineType{ContainerdContainerRuntime, DockerContainerRuntime}
+var SupportedK8sEngineTypes = []ContainerEngineType{ContainerdContainerRuntime, DockerContainerRuntime, CRIOContainerRuntime}
 
 type K8sContainerEngineSpec struct {
 	Endpoint string `json:"endpoint,omitempty"`
-	// +kubebuilder:validation:Enum:=containerd;docker-daemon
+	// +kubebuilder:validation:Enum:=containerd;docker-daemon;cri-o
 	EngineType ContainerEngineType `json:"engineType,omitempty"`
+
+	// +kubebuilder:default:=<>
+	// CRIO holds configuration values specific to the CRI-O container engine
+	CRIO CRIOSpec `json:"CRIO,omitempty"`
+}
+
+type CRIOSpec struct {
+	// StoragePath can be used to set the path used by CRI-O to store images on each node.
+	// This path will be mounted on the cluster scanner to provide access to the node's images.
+	// If the path does not match what CRI-O uses on the nodes, then images will not be found and scanned as expected.
+	// If not specified, the default location of CRI-O is used (/var/lib/containers/storage).
+	// +kubebuilder:validation:Optional
+	StoragePath string `json:"storagePath,omitempty"`
+
+	// StorageConfigPath can be used to set the path to the storage configuration file used by CRI-O (if any).
+	// If not specified, the default location for storage is used (/etc/containers/storage.conf).
+	// The files does not need to exist.
+	// See https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md for more information
+	// +kubebuilder:validation:Optional
+	StorageConfigPath string `json:"storageConfigPath,omitempty"`
+
+	// ConfigPath can be used to set the path to CRI-O's configuration file.
+	// If not specified, the default location is used (/etc/crio/crio.conf).
+	// See https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md for more information.
+	// +kubebuilder:validation:Optional
+	ConfigPath string `json:"configPath,omitempty"`
 }

cbcontainers/state/components/sensor_daemon_set.go

@@ -28,16 +28,30 @@ const (
 
 	desiredConnectionTimeoutSeconds = 60
 
-	// k8s container runtime/contaienr engine endpoints
+	// k8s container runtime/container engine endpoints
 	containerdRuntimeEndpoint         = "/var/run/containerd/containerd.sock"
 	microk8sContainerdRuntimeEndpoint = "/var/snap/microk8s/common/run/containerd.sock"
 	k3sContainerdRuntimeEndpoint      = "/run/k3s/containerd/containerd.sock"
 	dockerRuntimeEndpoint             = "/var/run/dockershim.sock"
 	dockerSock                        = "/var/run/docker.sock"
 	crioRuntimeEndpoint               = "/var/run/crio/crio.sock"
 
+	// configuredContainerRuntimeVolumeName is used when the customer has specified a non-standard runtime endpoint in the CRD
+	// as this means we need a special volume+mount for this endpoint
 	configuredContainerRuntimeVolumeName = "configured-container-runtime-endpoint"
 
+	// CRI-O specific volumes since the socket is not enough to read image blobs from the host
+
+	crioStorageVolumeName       = "crio-storage"
+	crioStorageConfigVolumeName = "crio-storage-config"
+	crioConfigVolumeName        = "crio-config"
+
+	// Source: https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md and https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md
+
+	crioStorageDefaultPath       = "/var/lib/containers"
+	crioStorageConfigDefaultPath = "/etc/containers/storage.conf"
+	crioConfigDefaultPath        = "/etc/crio/crio.conf"
+
 	cndrCompanyCodeVarName = "CB_COMPANY_CODES"
 	cndrCompanyCodeKeyName = "companyCode"
 )
@@ -200,6 +214,10 @@ func (obj *SensorDaemonSetK8sObject) getExpectedVolumeCount(agentSpec *cbContain
 		if clusterScannerSpec.K8sContainerEngine.Endpoint != "" {
 			expectedVolumesCount += 1
 		}
+
+		// cri-o specific mounts to use the containers/storage library with the host's storage area
+		// one mount is for the image store on the host, one is for the storage.conf and one is for crio.conf => we expect 3 more
+		expectedVolumesCount += 3
 	}
 
 	if isCndrEnbaled(agentSpec.Components.Cndr) {
@@ -474,7 +492,6 @@ func (obj *SensorDaemonSetK8sObject) mutateCndrVolumesMounts(container *coreV1.C
 }
 
 func (obj *SensorDaemonSetK8sObject) mutateClusterScannerEnvVars(container *coreV1.Container, agentSpec *cbContainersV1.CBContainersAgentSpec) {
-
 	clusterScannerSpec := &agentSpec.Components.ClusterScanning.ClusterScannerAgent
 
 	customEnvs := []coreV1.EnvVar{
@@ -529,14 +546,48 @@ func (obj *SensorDaemonSetK8sObject) mutateClusterScannerVolumes(templatePodSpec
 		templatePodSpec.Volumes[routeIndex].HostPath.Path = clusterScannerSpec.K8sContainerEngine.Endpoint
 	}
 
+	// Ensure we have volumes for CRI-O
+	crioStorageIx := commonState.EnsureAndGetVolumeIndexForName(templatePodSpec, crioStorageVolumeName)
+	storagePath := clusterScannerSpec.K8sContainerEngine.CRIO.StoragePath
+	if storagePath == "" {
+		storagePath = crioStorageDefaultPath
+	}
+	if templatePodSpec.Volumes[crioStorageIx].HostPath == nil {
+		templatePodSpec.Volumes[crioStorageIx].HostPath = &coreV1.HostPathVolumeSource{}
+	}
+	templatePodSpec.Volumes[crioStorageIx].HostPath.Path = storagePath
+
+	crioStorageConfigIx := commonState.EnsureAndGetVolumeIndexForName(templatePodSpec, crioStorageConfigVolumeName)
+	storageConfigPath := clusterScannerSpec.K8sContainerEngine.CRIO.StorageConfigPath
+	if storageConfigPath == "" {
+		storageConfigPath = crioStorageConfigDefaultPath
+	}
+	if templatePodSpec.Volumes[crioStorageConfigIx].HostPath == nil {
+		templatePodSpec.Volumes[crioStorageConfigIx].HostPath = &coreV1.HostPathVolumeSource{}
+	}
+	templatePodSpec.Volumes[crioStorageConfigIx].HostPath.Path = storageConfigPath
+
+	crioConfigIx := commonState.EnsureAndGetVolumeIndexForName(templatePodSpec, crioConfigVolumeName)
+	configPath := clusterScannerSpec.K8sContainerEngine.CRIO.ConfigPath
+	if configPath == "" {
+		configPath = crioConfigDefaultPath
+	}
+	if templatePodSpec.Volumes[crioConfigIx].HostPath == nil {
+		templatePodSpec.Volumes[crioConfigIx].HostPath = &coreV1.HostPathVolumeSource{}
+	}
+	templatePodSpec.Volumes[crioConfigIx].HostPath.Path = configPath
+	// End CRI-O config
+
 	// mutate root-cas volume, for https certificates
 	commonState.MutateVolumesToIncludeRootCAsVolume(templatePodSpec)
 }
 
 func (obj *SensorDaemonSetK8sObject) mutateClusterScannerVolumesMounts(container *coreV1.Container, agentSpec *cbContainersV1.CBContainersAgentSpec) {
 	containerRuntimes := getClusterScannerContainerRuntimes(&agentSpec.Components.ClusterScanning.ClusterScannerAgent)
 
-	if container.VolumeMounts == nil || len(container.VolumeMounts) != len(containerRuntimes)+1 {
+	// We expect to see
+	// container runtimes mounts + root CA mount + 3 mounts for CRI-O
+	if container.VolumeMounts == nil || len(container.VolumeMounts) != (len(containerRuntimes)+1+3) {
 		container.VolumeMounts = make([]coreV1.VolumeMount, 0)
 	}
 
@@ -548,6 +599,15 @@ func (obj *SensorDaemonSetK8sObject) mutateClusterScannerVolumesMounts(container
 		index := commonState.EnsureAndGetVolumeMountIndexForName(container, name)
 		commonState.MutateVolumeMount(container, index, mountPath, true)
 	}
+
+	// mutate mounts for the CRI-O engine storage
+	crioStorageIx := commonState.EnsureAndGetVolumeMountIndexForName(container, crioStorageVolumeName)
+	// The storage MUST be R/W as file-based locks are used to enable concurrent access to the same storage from both CRI-O and our agent
+	commonState.MutateVolumeMount(container, crioStorageIx, crioStorageDefaultPath, false)
+	crioStorageConfigIx := commonState.EnsureAndGetVolumeMountIndexForName(container, crioStorageConfigVolumeName)
+	commonState.MutateVolumeMount(container, crioStorageConfigIx, crioStorageConfigDefaultPath, true)
+	crioConfigIx := commonState.EnsureAndGetVolumeMountIndexForName(container, crioConfigVolumeName)
+	commonState.MutateVolumeMount(container, crioConfigIx, crioConfigDefaultPath, true)
 }
 
 // The cluster scanner uses the container runtime unix socket to fetch the running containers to scan.

charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml

@@ -1845,12 +1845,44 @@ spec:
                           k8sContainerEngine:
                             default: {}
                             properties:
+                              CRIO:
+                                default: { }
+                                description: CRIO holds configuration values specific
+                                  to the CRI-O container engine
+                                properties:
+                                  configPath:
+                                    description: ConfigPath can be used to set the
+                                      path to CRI-O's configuration file. If not specified,
+                                      the default location is used (/etc/crio/crio.conf).
+                                      See https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md
+                                      for more information.
+                                    type: string
+                                  storageConfigPath:
+                                    description: StorageConfigPath can be used to
+                                      set the path to the storage configuration file
+                                      used by CRI-O (if any). If not specified, the
+                                      default location for storage is used (/etc/containers/storage.conf).
+                                      The files does not need to exist. See https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md
+                                      for more information
+                                    type: string
+                                  storagePath:
+                                    description: StoragePath can be used to set the
+                                      path used by CRI-O to store images on each node.
+                                      This path will be mounted on the cluster scanner
+                                      to provide access to the node's images. If the
+                                      path does not match what CRI-O uses on the nodes,
+                                      then images will not be found and scanned as
+                                      expected. If not specified, the default location
+                                      of CRI-O is used (/var/lib/containers/storage).
+                                    type: string
+                                type: object
                               endpoint:
                                 type: string
                               engineType:
                                 enum:
                                 - containerd
                                 - docker-daemon
+                                - cri-o
                                 type: string
                             type: object
                           labels:

config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml

@@ -3346,12 +3346,44 @@ spec:
                           k8sContainerEngine:
                             default: {}
                             properties:
+                              CRIO:
+                                default: {}
+                                description: CRIO holds configuration values specific
+                                  to the CRI-O container engine
+                                properties:
+                                  configPath:
+                                    description: ConfigPath can be used to set the
+                                      path to CRI-O's configuration file. If not specified,
+                                      the default location is used (/etc/crio/crio.conf).
+                                      See https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md
+                                      for more information.
+                                    type: string
+                                  storageConfigPath:
+                                    description: StorageConfigPath can be used to
+                                      set the path to the storage configuration file
+                                      used by CRI-O (if any). If not specified, the
+                                      default location for storage is used (/etc/containers/storage.conf).
+                                      The files does not need to exist. See https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md
+                                      for more information
+                                    type: string
+                                  storagePath:
+                                    description: StoragePath can be used to set the
+                                      path used by CRI-O to store images on each node.
+                                      This path will be mounted on the cluster scanner
+                                      to provide access to the node's images. If the
+                                      path does not match what CRI-O uses on the nodes,
+                                      then images will not be found and scanned as
+                                      expected. If not specified, the default location
+                                      of CRI-O is used (/var/lib/containers/storage).
+                                    type: string
+                                type: object
                               endpoint:
                                 type: string
                               engineType:
                                 enum:
                                 - containerd
                                 - docker-daemon
+                                - cri-o
                                 type: string
                             type: object
                           labels:

config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml

@@ -3140,12 +3140,43 @@ spec:
                           type: object
                         k8sContainerEngine:
                           properties:
+                            CRIO:
+                              description: CRIO holds configuration values specific
+                                to the CRI-O container engine
+                              properties:
+                                configPath:
+                                  description: ConfigPath can be used to set the path
+                                    to CRI-O's configuration file. If not specified,
+                                    the default location is used (/etc/crio/crio.conf).
+                                    See https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md
+                                    for more information.
+                                  type: string
+                                storageConfigPath:
+                                  description: StorageConfigPath can be used to set
+                                    the path to the storage configuration file used
+                                    by CRI-O (if any). If not specified, the default
+                                    location for storage is used (/etc/containers/storage.conf).
+                                    The files does not need to exist. See https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md
+                                    for more information
+                                  type: string
+                                storagePath:
+                                  description: StoragePath can be used to set the
+                                    path used by CRI-O to store images on each node.
+                                    This path will be mounted on the cluster scanner
+                                    to provide access to the node's images. If the
+                                    path does not match what CRI-O uses on the nodes,
+                                    then images will not be found and scanned as expected.
+                                    If not specified, the default location of CRI-O
+                                    is used (/var/lib/containers/storage).
+                                  type: string
+                              type: object
                             endpoint:
                               type: string
                             engineType:
                               enum:
                               - containerd
                               - docker-daemon
+                              - cri-o
                               type: string
                           type: object
                         labels:

controllers/cbcontainersagent_controller.go

@@ -98,7 +98,7 @@ func (r *CBContainersAgentController) Reconcile(ctx context.Context, req ctrl.Re
 	}
 
 	if err := r.setAgentDefaults(&cbContainersAgent.Spec); err != nil {
-		return ctrl.Result{}, fmt.Errorf("faild to set defaults to cluster CR: %v", err)
+		return ctrl.Result{}, fmt.Errorf("failed to set defaults to cluster CR: %v", err)
 	}
 
 	setOwner := func(controlledResource metav1.Object) error {

controllers/cluster_scanning_components_defaults.go

@@ -87,30 +87,34 @@ func (r *CBContainersAgentController) setClusterScannerAgentDefaults(clusterScan
 
 	setDefaultFileProbes(&clusterScannerAgent.Probes)
 
-	emptyK8sContainerEngineSpec := cbcontainersv1.K8sContainerEngineSpec{}
-	if clusterScannerAgent.K8sContainerEngine != emptyK8sContainerEngineSpec {
-		if err := validateClusterScannerK8sContainerEngineSpec(clusterScannerAgent.K8sContainerEngine); err != nil {
-			return err
-		}
+	if err := validateClusterScannerK8sContainerEngineSpec(clusterScannerAgent.K8sContainerEngine); err != nil {
+		return err
 	}
 
 	return nil
 }
 
 func validateClusterScannerK8sContainerEngineSpec(spec cbcontainersv1.K8sContainerEngineSpec) error {
-	if spec.Endpoint == "" {
-		return fmt.Errorf("k8s container engine endpoint must be provided if configuring k8s container engine option")
+	if spec.Endpoint == "" && spec.EngineType != "" {
+		return fmt.Errorf("k8s container engine endpoint must be provided if the k8s container engine type has been set")
 	}
 
-	if spec.EngineType == "" {
-		return fmt.Errorf("k8s container engine type must be provided if configuring k8s container engine option")
+	if spec.Endpoint != "" && spec.EngineType == "" {
+		return fmt.Errorf("k8s container engine type must be provided if the container endpoint has been set")
 	}
 
-	for _, engineType := range cbcontainersv1.SupportedK8sEngineTypes {
-		if engineType == spec.EngineType {
-			return nil
+	if spec.EngineType != "" {
+		found := false
+		for _, engineType := range cbcontainersv1.SupportedK8sEngineTypes {
+			if engineType == spec.EngineType {
+				found = true
+				break
+			}
+		}
+		if !found {
+			return fmt.Errorf("invalid engine type %v provided", spec.EngineType)
 		}
 	}
 
-	return fmt.Errorf("invalid engine type %v provided", spec.EngineType)
+	return nil
 }