From 08a0769a36cb7ab96fcdf9e4c01198399159bfcf Mon Sep 17 00:00:00 2001 From: ltsonov Date: Wed, 31 May 2023 15:31:13 +0300 Subject: [PATCH 1/9] Add config values for CRIO mounts --- api/v1/container_runtime_types.go | 24 +++++++++++++++++-- ...ers.carbonblack.io_cbcontainersagents.yaml | 24 +++++++++++++++++++ ...ers.carbonblack.io_cbcontainersagents.yaml | 21 ++++++++++++++++ .../cluster_scanning_components_defaults.go | 14 +++++++++++ 4 files changed, 81 insertions(+), 2 deletions(-) diff --git a/api/v1/container_runtime_types.go b/api/v1/container_runtime_types.go index 84d0d5db..37d6af3c 100644 --- a/api/v1/container_runtime_types.go +++ b/api/v1/container_runtime_types.go @@ -12,10 +12,30 @@ const ( CRIOContainerRuntime ContainerEngineType = "cri-o" ) -var SupportedK8sEngineTypes = []ContainerEngineType{ContainerdContainerRuntime, DockerContainerRuntime} +var SupportedK8sEngineTypes = []ContainerEngineType{ContainerdContainerRuntime, DockerContainerRuntime, CRIOContainerRuntime} type K8sContainerEngineSpec struct { Endpoint string `json:"endpoint,omitempty"` - // +kubebuilder:validation:Enum:=containerd;docker-daemon + // +kubebuilder:validation:Enum:=containerd;docker-daemon;cri-o EngineType ContainerEngineType `json:"engineType,omitempty"` + + // +kubebuilder:default:=<> + // CRIO holds configuration values specific to the CRI-O container engine + CRIO CRIOSpec `json:"CRIO,omitempty"` +} + +type CRIOSpec struct { + // StoragePath can be used to set the path used by CRI-O to store images on each node. + // This path will be mounted on the cluster scanner to provide access to the node's images. + // If the path does not match what CRI-O uses on the nodes, then images will not be found and scanned as expected. + // If not specified, the default location of CRI-O is used. + // +kubebuilder:validation:Optional + // +kubebuilder:default:="/var/lib/containers/storage" + StoragePath string `json:"storagePath,omitempty"` + + // ConfigPath can be used to set the path to CRI-O's configuration file + // If not specified, the default location for CRI-O is used. + // +kubebuilder:validation:Optional + // +kubebuilder:default:="/etc/containers/storage.conf" + ConfigPath string `json:"configPath,omitempty"` } diff --git a/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml b/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml index e1708bbd..fde095d6 100644 --- a/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml +++ b/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml @@ -3346,12 +3346,36 @@ spec: k8sContainerEngine: default: {} properties: + CRIO: + default: {} + description: CRIO holds configuration values specific + to the CRI-O container engine + properties: + configPath: + default: /etc/containers/storage.conf + description: ConfigPath can be used to set the + path to CRI-O's configuration file If not specified, + the default location for CRI-O is used. + type: string + storagePath: + default: /var/lib/containers/storage + description: StoragePath can be used to set the + path used by CRI-O to store images on each node. + This path will be mounted on the cluster scanner + to provide access to the node's images. If the + path does not match what CRI-O uses on the nodes, + then images will not be found and scanned as + expected. If not specified, the default location + of CRI-O is used. + type: string + type: object endpoint: type: string engineType: enum: - containerd - docker-daemon + - cri-o type: string type: object labels: diff --git a/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml b/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml index d3bd0260..86305db0 100644 --- a/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml +++ b/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml @@ -3140,12 +3140,33 @@ spec: type: object k8sContainerEngine: properties: + CRIO: + description: CRIO holds configuration values specific + to the CRI-O container engine + properties: + configPath: + description: ConfigPath can be used to set the path + to CRI-O's configuration file If not specified, + the default location for CRI-O is used. + type: string + storagePath: + description: StoragePath can be used to set the + path used by CRI-O to store images on each node. + This path will be mounted on the cluster scanner + to provide access to the node's images. If the + path does not match what CRI-O uses on the nodes, + then images will not be found and scanned as expected. + If not specified, the default location of CRI-O + is used. + type: string + type: object endpoint: type: string engineType: enum: - containerd - docker-daemon + - cri-o type: string type: object labels: diff --git a/controllers/cluster_scanning_components_defaults.go b/controllers/cluster_scanning_components_defaults.go index b6fc1c4f..54a910dd 100644 --- a/controllers/cluster_scanning_components_defaults.go +++ b/controllers/cluster_scanning_components_defaults.go @@ -5,6 +5,13 @@ import ( cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" ) +const ( + // Source: https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md + + crioDefaultStoragePathCRD = "/var/lib/containers/storage" + crioDefaultConfigPathCRD = "/etc/containers/storage.conf" +) + func (r *CBContainersAgentController) setClusterScanningComponentsDefaults(clusterScanning *cbcontainersv1.CBContainersClusterScanningSpec) error { if clusterScanning.Enabled == nil { clusterScanning.Enabled = &trueRef @@ -94,6 +101,13 @@ func (r *CBContainersAgentController) setClusterScannerAgentDefaults(clusterScan } } + if clusterScannerAgent.K8sContainerEngine.CRIO.ConfigPath == "" { + clusterScannerAgent.K8sContainerEngine.CRIO.ConfigPath = crioDefaultConfigPathCRD + } + if clusterScannerAgent.K8sContainerEngine.CRIO.StoragePath == "" { + clusterScannerAgent.K8sContainerEngine.CRIO.StoragePath = crioDefaultStoragePathCRD + } + return nil } From 0727e286e79056942d32f0188b208925525e90b2 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Wed, 31 May 2023 15:36:56 +0300 Subject: [PATCH 2/9] Setup CRIO mounts on the node-agent based on config values --- .../state/components/sensor_daemon_set.go | 38 ++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/cbcontainers/state/components/sensor_daemon_set.go b/cbcontainers/state/components/sensor_daemon_set.go index 464ed48a..79495427 100644 --- a/cbcontainers/state/components/sensor_daemon_set.go +++ b/cbcontainers/state/components/sensor_daemon_set.go @@ -28,7 +28,7 @@ const ( desiredConnectionTimeoutSeconds = 60 - // k8s container runtime/contaienr engine endpoints + // k8s container runtime/container engine endpoints containerdRuntimeEndpoint = "/var/run/containerd/containerd.sock" microk8sContainerdRuntimeEndpoint = "/var/snap/microk8s/common/run/containerd.sock" k3sContainerdRuntimeEndpoint = "/run/k3s/containerd/containerd.sock" @@ -36,8 +36,17 @@ const ( dockerSock = "/var/run/docker.sock" crioRuntimeEndpoint = "/var/run/crio/crio.sock" + // configuredContainerRuntimeVolumeName is used when the customer has specified a non-standard runtime endpoint in the CRD + // as this means we need a special volume+mount for this endpoint configuredContainerRuntimeVolumeName = "configured-container-runtime-endpoint" + // CRI-O specific volumes since the socket is not enough to read image blobs from the host + + crioStorageVolumeName = "crio-storage" + crioConfigVolumeName = "crio-config" + crioStorageMountedPath = "/var/lib/containers/storage" + crioConfigMountedPath = "/etc/containers/storage.conf" + cndrCompanyCodeVarName = "CB_COMPANY_CODES" cndrCompanyCodeKeyName = "companyCode" ) @@ -200,6 +209,10 @@ func (obj *SensorDaemonSetK8sObject) getExpectedVolumeCount(agentSpec *cbContain if clusterScannerSpec.K8sContainerEngine.Endpoint != "" { expectedVolumesCount += 1 } + + // cri-o specific mounts to use the containers/storage library with the host's storage area + // one mount is for the image store on the host and one is for the storage.config => we expect 2 more + expectedVolumesCount += 2 } if isCndrEnbaled(agentSpec.Components.Cndr) { @@ -529,6 +542,22 @@ func (obj *SensorDaemonSetK8sObject) mutateClusterScannerVolumes(templatePodSpec templatePodSpec.Volumes[routeIndex].HostPath.Path = clusterScannerSpec.K8sContainerEngine.Endpoint } + // Ensure we have volumes for CRI-O + crioStorageIx := commonState.EnsureAndGetVolumeIndexForName(templatePodSpec, crioStorageVolumeName) + if clusterScannerSpec.K8sContainerEngine.CRIO.StoragePath != "" { + if templatePodSpec.Volumes[crioStorageIx].HostPath == nil { + templatePodSpec.Volumes[crioStorageIx].HostPath = &coreV1.HostPathVolumeSource{} + } + templatePodSpec.Volumes[crioStorageIx].HostPath.Path = clusterScannerSpec.K8sContainerEngine.CRIO.StoragePath + } + crioConfigIx := commonState.EnsureAndGetVolumeIndexForName(templatePodSpec, crioConfigVolumeName) + if clusterScannerSpec.K8sContainerEngine.CRIO.ConfigPath != "" { + if templatePodSpec.Volumes[crioConfigIx].HostPath == nil { + templatePodSpec.Volumes[crioConfigIx].HostPath = &coreV1.HostPathVolumeSource{} + } + templatePodSpec.Volumes[crioConfigIx].HostPath.Path = clusterScannerSpec.K8sContainerEngine.CRIO.ConfigPath + } + // mutate root-cas volume, for https certificates commonState.MutateVolumesToIncludeRootCAsVolume(templatePodSpec) } @@ -548,6 +577,13 @@ func (obj *SensorDaemonSetK8sObject) mutateClusterScannerVolumesMounts(container index := commonState.EnsureAndGetVolumeMountIndexForName(container, name) commonState.MutateVolumeMount(container, index, mountPath, true) } + + // mutate mounts for the CRI-O engine storage + crioStorageIx := commonState.EnsureAndGetVolumeMountIndexForName(container, crioStorageVolumeName) + // The storage MUST be R/W as file-based locks are used to enable concurrent access to the same storage from both CRI-O and our agent + commonState.MutateVolumeMount(container, crioStorageIx, crioStorageMountedPath, false) + crioConfigIx := commonState.EnsureAndGetVolumeMountIndexForName(container, crioConfigVolumeName) + commonState.MutateVolumeMount(container, crioConfigIx, crioConfigMountedPath, true) } // The cluster scanner uses the container runtime unix socket to fetch the running containers to scan. From 68503c8d0fc04bb0ef19e2859e8a06cf7d79a416 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Thu, 1 Jun 2023 10:11:20 +0300 Subject: [PATCH 3/9] Change CRIO config so it is optional and defaults are not set in the CR. Instead, values can be used to override the defaults from code, similar to how the rest of the Engine spec works. --- api/v1/container_runtime_types.go | 6 +-- api/v1/zz_generated.deepcopy.go | 16 +++++++ ...ers.carbonblack.io_cbcontainersagents.yaml | 6 +-- ...ers.carbonblack.io_cbcontainersagents.yaml | 4 +- .../cluster_scanning_components_defaults.go | 44 +++++++------------ 5 files changed, 39 insertions(+), 37 deletions(-) diff --git a/api/v1/container_runtime_types.go b/api/v1/container_runtime_types.go index 37d6af3c..8fdc7259 100644 --- a/api/v1/container_runtime_types.go +++ b/api/v1/container_runtime_types.go @@ -28,14 +28,12 @@ type CRIOSpec struct { // StoragePath can be used to set the path used by CRI-O to store images on each node. // This path will be mounted on the cluster scanner to provide access to the node's images. // If the path does not match what CRI-O uses on the nodes, then images will not be found and scanned as expected. - // If not specified, the default location of CRI-O is used. + // If not specified, the default location of CRI-O is used (/var/lib/containers/storage). // +kubebuilder:validation:Optional - // +kubebuilder:default:="/var/lib/containers/storage" StoragePath string `json:"storagePath,omitempty"` // ConfigPath can be used to set the path to CRI-O's configuration file - // If not specified, the default location for CRI-O is used. + // If not specified, the default location for CRI-O is used (/etc/containers/storage.conf). // +kubebuilder:validation:Optional - // +kubebuilder:default:="/etc/containers/storage.conf" ConfigPath string `json:"configPath,omitempty"` } diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 2b92a496..960b55c2 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -877,9 +877,25 @@ func (in *CBContainersStateReporterSpec) DeepCopy() *CBContainersStateReporterSp return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CRIOSpec) DeepCopyInto(out *CRIOSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CRIOSpec. +func (in *CRIOSpec) DeepCopy() *CRIOSpec { + if in == nil { + return nil + } + out := new(CRIOSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *K8sContainerEngineSpec) DeepCopyInto(out *K8sContainerEngineSpec) { *out = *in + out.CRIO = in.CRIO } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new K8sContainerEngineSpec. diff --git a/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml b/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml index fde095d6..84fdcd10 100644 --- a/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml +++ b/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml @@ -3352,13 +3352,11 @@ spec: to the CRI-O container engine properties: configPath: - default: /etc/containers/storage.conf description: ConfigPath can be used to set the path to CRI-O's configuration file If not specified, - the default location for CRI-O is used. + the default location for CRI-O is used (/etc/containers/storage.conf). type: string storagePath: - default: /var/lib/containers/storage description: StoragePath can be used to set the path used by CRI-O to store images on each node. This path will be mounted on the cluster scanner @@ -3366,7 +3364,7 @@ spec: path does not match what CRI-O uses on the nodes, then images will not be found and scanned as expected. If not specified, the default location - of CRI-O is used. + of CRI-O is used (/var/lib/containers/storage). type: string type: object endpoint: diff --git a/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml b/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml index 86305db0..db04bc93 100644 --- a/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml +++ b/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml @@ -3147,7 +3147,7 @@ spec: configPath: description: ConfigPath can be used to set the path to CRI-O's configuration file If not specified, - the default location for CRI-O is used. + the default location for CRI-O is used (/etc/containers/storage.conf). type: string storagePath: description: StoragePath can be used to set the @@ -3157,7 +3157,7 @@ spec: path does not match what CRI-O uses on the nodes, then images will not be found and scanned as expected. If not specified, the default location of CRI-O - is used. + is used (/var/lib/containers/storage). type: string type: object endpoint: diff --git a/controllers/cluster_scanning_components_defaults.go b/controllers/cluster_scanning_components_defaults.go index 54a910dd..cdc0d682 100644 --- a/controllers/cluster_scanning_components_defaults.go +++ b/controllers/cluster_scanning_components_defaults.go @@ -5,13 +5,6 @@ import ( cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" ) -const ( - // Source: https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md - - crioDefaultStoragePathCRD = "/var/lib/containers/storage" - crioDefaultConfigPathCRD = "/etc/containers/storage.conf" -) - func (r *CBContainersAgentController) setClusterScanningComponentsDefaults(clusterScanning *cbcontainersv1.CBContainersClusterScanningSpec) error { if clusterScanning.Enabled == nil { clusterScanning.Enabled = &trueRef @@ -94,37 +87,34 @@ func (r *CBContainersAgentController) setClusterScannerAgentDefaults(clusterScan setDefaultFileProbes(&clusterScannerAgent.Probes) - emptyK8sContainerEngineSpec := cbcontainersv1.K8sContainerEngineSpec{} - if clusterScannerAgent.K8sContainerEngine != emptyK8sContainerEngineSpec { - if err := validateClusterScannerK8sContainerEngineSpec(clusterScannerAgent.K8sContainerEngine); err != nil { - return err - } - } - - if clusterScannerAgent.K8sContainerEngine.CRIO.ConfigPath == "" { - clusterScannerAgent.K8sContainerEngine.CRIO.ConfigPath = crioDefaultConfigPathCRD - } - if clusterScannerAgent.K8sContainerEngine.CRIO.StoragePath == "" { - clusterScannerAgent.K8sContainerEngine.CRIO.StoragePath = crioDefaultStoragePathCRD + if err := validateClusterScannerK8sContainerEngineSpec(clusterScannerAgent.K8sContainerEngine); err != nil { + return err } return nil } func validateClusterScannerK8sContainerEngineSpec(spec cbcontainersv1.K8sContainerEngineSpec) error { - if spec.Endpoint == "" { - return fmt.Errorf("k8s container engine endpoint must be provided if configuring k8s container engine option") + if spec.Endpoint == "" && spec.EngineType != "" { + return fmt.Errorf("k8s container engine endpoint must be provided if the k8s container engine type has been set") } - if spec.EngineType == "" { - return fmt.Errorf("k8s container engine type must be provided if configuring k8s container engine option") + if spec.Endpoint != "" && spec.EngineType == "" { + return fmt.Errorf("k8s container engine type must be provided if the container endpoint has been set") } - for _, engineType := range cbcontainersv1.SupportedK8sEngineTypes { - if engineType == spec.EngineType { - return nil + if spec.EngineType != "" { + found := false + for _, engineType := range cbcontainersv1.SupportedK8sEngineTypes { + if engineType == spec.EngineType { + found = true + break + } + } + if !found { + return fmt.Errorf("invalid engine type %v provided", spec.EngineType) } } - return fmt.Errorf("invalid engine type %v provided", spec.EngineType) + return nil } From 10a1d347f62d66712ed31bb90c1115c84f30c9b7 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Thu, 1 Jun 2023 11:01:54 +0300 Subject: [PATCH 4/9] Set default CRIO mounts in reconcile loop --- .../state/components/sensor_daemon_set.go | 39 +++++++++++-------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/cbcontainers/state/components/sensor_daemon_set.go b/cbcontainers/state/components/sensor_daemon_set.go index 79495427..d64f959d 100644 --- a/cbcontainers/state/components/sensor_daemon_set.go +++ b/cbcontainers/state/components/sensor_daemon_set.go @@ -42,10 +42,12 @@ const ( // CRI-O specific volumes since the socket is not enough to read image blobs from the host - crioStorageVolumeName = "crio-storage" - crioConfigVolumeName = "crio-config" - crioStorageMountedPath = "/var/lib/containers/storage" - crioConfigMountedPath = "/etc/containers/storage.conf" + crioStorageVolumeName = "crio-storage" + crioConfigVolumeName = "crio-config" + // Source: https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md + + crioStorageDefaultPath = "/var/lib/containers/storage" + crioConfigDefaultPath = "/etc/containers/storage.conf" cndrCompanyCodeVarName = "CB_COMPANY_CODES" cndrCompanyCodeKeyName = "companyCode" @@ -544,19 +546,24 @@ func (obj *SensorDaemonSetK8sObject) mutateClusterScannerVolumes(templatePodSpec // Ensure we have volumes for CRI-O crioStorageIx := commonState.EnsureAndGetVolumeIndexForName(templatePodSpec, crioStorageVolumeName) - if clusterScannerSpec.K8sContainerEngine.CRIO.StoragePath != "" { - if templatePodSpec.Volumes[crioStorageIx].HostPath == nil { - templatePodSpec.Volumes[crioStorageIx].HostPath = &coreV1.HostPathVolumeSource{} - } - templatePodSpec.Volumes[crioStorageIx].HostPath.Path = clusterScannerSpec.K8sContainerEngine.CRIO.StoragePath + storagePath := clusterScannerSpec.K8sContainerEngine.CRIO.StoragePath + if storagePath == "" { + storagePath = crioStorageDefaultPath + } + if templatePodSpec.Volumes[crioStorageIx].HostPath == nil { + templatePodSpec.Volumes[crioStorageIx].HostPath = &coreV1.HostPathVolumeSource{} } + templatePodSpec.Volumes[crioStorageIx].HostPath.Path = storagePath + crioConfigIx := commonState.EnsureAndGetVolumeIndexForName(templatePodSpec, crioConfigVolumeName) - if clusterScannerSpec.K8sContainerEngine.CRIO.ConfigPath != "" { - if templatePodSpec.Volumes[crioConfigIx].HostPath == nil { - templatePodSpec.Volumes[crioConfigIx].HostPath = &coreV1.HostPathVolumeSource{} - } - templatePodSpec.Volumes[crioConfigIx].HostPath.Path = clusterScannerSpec.K8sContainerEngine.CRIO.ConfigPath + configPath := clusterScannerSpec.K8sContainerEngine.CRIO.ConfigPath + if configPath == "" { + configPath = crioConfigDefaultPath + } + if templatePodSpec.Volumes[crioConfigIx].HostPath == nil { + templatePodSpec.Volumes[crioConfigIx].HostPath = &coreV1.HostPathVolumeSource{} } + templatePodSpec.Volumes[crioConfigIx].HostPath.Path = configPath // mutate root-cas volume, for https certificates commonState.MutateVolumesToIncludeRootCAsVolume(templatePodSpec) @@ -581,9 +588,9 @@ func (obj *SensorDaemonSetK8sObject) mutateClusterScannerVolumesMounts(container // mutate mounts for the CRI-O engine storage crioStorageIx := commonState.EnsureAndGetVolumeMountIndexForName(container, crioStorageVolumeName) // The storage MUST be R/W as file-based locks are used to enable concurrent access to the same storage from both CRI-O and our agent - commonState.MutateVolumeMount(container, crioStorageIx, crioStorageMountedPath, false) + commonState.MutateVolumeMount(container, crioStorageIx, crioStorageDefaultPath, false) crioConfigIx := commonState.EnsureAndGetVolumeMountIndexForName(container, crioConfigVolumeName) - commonState.MutateVolumeMount(container, crioConfigIx, crioConfigMountedPath, true) + commonState.MutateVolumeMount(container, crioConfigIx, crioConfigDefaultPath, true) } // The cluster scanner uses the container runtime unix socket to fetch the running containers to scan. From 5a68075fc71e7a14bb27399a17eff4b2cb5a07ec Mon Sep 17 00:00:00 2001 From: ltsonov Date: Thu, 1 Jun 2023 11:03:48 +0300 Subject: [PATCH 5/9] Spellings --- controllers/cbcontainersagent_controller.go | 2 +- docs/developers.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/controllers/cbcontainersagent_controller.go b/controllers/cbcontainersagent_controller.go index 46d41548..b367844a 100644 --- a/controllers/cbcontainersagent_controller.go +++ b/controllers/cbcontainersagent_controller.go @@ -98,7 +98,7 @@ func (r *CBContainersAgentController) Reconcile(ctx context.Context, req ctrl.Re } if err := r.setAgentDefaults(&cbContainersAgent.Spec); err != nil { - return ctrl.Result{}, fmt.Errorf("faild to set defaults to cluster CR: %v", err) + return ctrl.Result{}, fmt.Errorf("failed to set defaults to cluster CR: %v", err) } setOwner := func(controlledResource metav1.Object) error { diff --git a/docs/developers.md b/docs/developers.md index d78afed4..f9bd6e07 100644 --- a/docs/developers.md +++ b/docs/developers.md @@ -2,7 +2,7 @@ ## Developer Guide -SKD version = 1.27.0 +SDK version = 1.27.0 ### Deploying the operator without using an image From a5004fd873880dc5a02f3e257f0b8048b1de96a8 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Thu, 1 Jun 2023 12:04:32 +0300 Subject: [PATCH 6/9] Fix infinite reconcile --- cbcontainers/state/components/sensor_daemon_set.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cbcontainers/state/components/sensor_daemon_set.go b/cbcontainers/state/components/sensor_daemon_set.go index d64f959d..ef35e946 100644 --- a/cbcontainers/state/components/sensor_daemon_set.go +++ b/cbcontainers/state/components/sensor_daemon_set.go @@ -572,7 +572,9 @@ func (obj *SensorDaemonSetK8sObject) mutateClusterScannerVolumes(templatePodSpec func (obj *SensorDaemonSetK8sObject) mutateClusterScannerVolumesMounts(container *coreV1.Container, agentSpec *cbContainersV1.CBContainersAgentSpec) { containerRuntimes := getClusterScannerContainerRuntimes(&agentSpec.Components.ClusterScanning.ClusterScannerAgent) - if container.VolumeMounts == nil || len(container.VolumeMounts) != len(containerRuntimes)+1 { + // We expect to see + // container runtimes mounts + root CA mount + 2 mounts for CRI-O + if container.VolumeMounts == nil || len(container.VolumeMounts) != (len(containerRuntimes)+1+2) { container.VolumeMounts = make([]coreV1.VolumeMount, 0) } From c4d7a012ed5e5c33e8563878fd7bec962f7781e2 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Thu, 1 Jun 2023 12:10:18 +0300 Subject: [PATCH 7/9] Add new params to docs --- docs/crds.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/crds.md b/docs/crds.md index 50f7a9fb..12358de8 100644 --- a/docs/crds.md +++ b/docs/crds.md @@ -58,15 +58,17 @@ This is the CR you'll need to deploy in order to trigger the operator to deploy ### Cluster Scanning Components Optional parameters -| Parameter | Description | Default | -|--------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------| -| `spec.components.clusterScanning.enabled` | Carbon Black Container flag to control Cluster Scanning components deployment | true | -| `spec.components.clusterScanning.imageScanningReporter.image.repository` | Carbon Black Container Image Scanning Reporter image repository | `cbartifactory/image-scanning-reporter` | -| `spec.components.clusterScanning.clusterScanner.image.repository` | Carbon Black Container Cluster Scanner Agent image repository | `cbartifactory/cluster-scanner` | -| `spec.components.clusterScanning.imageScanningReporter.resources` | Carbon Black Container Image Scanning Reporter resources | `{requests: {memory: "64Mi", cpu: "200m"}, limits: {memory: "1024Mi", cpu: "900m"}}` | -| `spec.components.clusterScanning.clusterScanner.resources` | Carbon Black Container Cluster Scanner resources | `{requests: {memory: "64Mi", cpu: "30m"}, limits: {memory: "1024Mi", cpu: "500m"}}` | -| `spec.components.clusterScanning.clusterScanner.k8sContainerEngine.engineType` | Carbon Black Container Cluster Scanner k8s container engine type. One of the options: `containerd`/`docker-daemon` | | -| `spec.components.clusterScanning.clusterScanner.k8sContainerEngine.endpoint` | Carbon Black Container Cluster Scanner k8s container engine endpoint path | | +| Parameter | Description | Default | +|--------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------| +| `spec.components.clusterScanning.enabled` | Carbon Black Container flag to control Cluster Scanning components deployment | true | +| `spec.components.clusterScanning.imageScanningReporter.image.repository` | Carbon Black Container Image Scanning Reporter image repository | `cbartifactory/image-scanning-reporter` | +| `spec.components.clusterScanning.clusterScanner.image.repository` | Carbon Black Container Cluster Scanner Agent image repository | `cbartifactory/cluster-scanner` | +| `spec.components.clusterScanning.imageScanningReporter.resources` | Carbon Black Container Image Scanning Reporter resources | `{requests: {memory: "64Mi", cpu: "200m"}, limits: {memory: "1024Mi", cpu: "900m"}}` | +| `spec.components.clusterScanning.clusterScanner.resources` | Carbon Black Container Cluster Scanner resources | `{requests: {memory: "64Mi", cpu: "30m"}, limits: {memory: "1024Mi", cpu: "500m"}}` | +| `spec.components.clusterScanning.clusterScanner.k8sContainerEngine.engineType` | Carbon Black Container Cluster Scanner k8s container engine type. One of the options: `containerd`/`docker-daemon`/`cri-o` | | +| `spec.components.clusterScanning.clusterScanner.k8sContainerEngine.endpoint` | Carbon Black Container Cluster Scanner k8s container engine endpoint path | | +| `spec.components.clusterScanning.clusterScanner.k8sContainerEngine.CRIO.storagePath` | Carbon Black Container Cluster Scanner override default image storage path (CRI-O only) | | +| `spec.components.clusterScanning.clusterScanner.k8sContainerEngine.CRIO.configPath` | Carbon Black Container Cluster Scanner override default image storage config path (CRI-O only) | | ### Components Common Optional parameters From 6ebd481ca455bcee3defa39adfe7af50bbc9398e Mon Sep 17 00:00:00 2001 From: ltsonov Date: Tue, 6 Jun 2023 12:39:43 +0300 Subject: [PATCH 8/9] Mount both storage.conf and crio.conf on the agent --- api/v1/container_runtime_types.go | 12 +++++-- .../state/components/sensor_daemon_set.go | 35 +++++++++++++------ ...ers.carbonblack.io_cbcontainersagents.yaml | 14 ++++++-- ...ers.carbonblack.io_cbcontainersagents.yaml | 14 ++++++-- 4 files changed, 59 insertions(+), 16 deletions(-) diff --git a/api/v1/container_runtime_types.go b/api/v1/container_runtime_types.go index 8fdc7259..4f854ead 100644 --- a/api/v1/container_runtime_types.go +++ b/api/v1/container_runtime_types.go @@ -32,8 +32,16 @@ type CRIOSpec struct { // +kubebuilder:validation:Optional StoragePath string `json:"storagePath,omitempty"` - // ConfigPath can be used to set the path to CRI-O's configuration file - // If not specified, the default location for CRI-O is used (/etc/containers/storage.conf). + // StorageConfigPath can be used to set the path to the storage configuration file used by CRI-O (if any). + // If not specified, the default location for storage is used (/etc/containers/storage.conf). + // The files does not need to exist. + // See https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md for more information + // +kubebuilder:validation:Optional + StorageConfigPath string `json:"storageConfigPath,omitempty"` + + // ConfigPath can be used to set the path to CRI-O's configuration file. + // If not specified, the default location is used (/etc/crio/crio.conf). + // See https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md for more information. // +kubebuilder:validation:Optional ConfigPath string `json:"configPath,omitempty"` } diff --git a/cbcontainers/state/components/sensor_daemon_set.go b/cbcontainers/state/components/sensor_daemon_set.go index ef35e946..f790783c 100644 --- a/cbcontainers/state/components/sensor_daemon_set.go +++ b/cbcontainers/state/components/sensor_daemon_set.go @@ -42,12 +42,15 @@ const ( // CRI-O specific volumes since the socket is not enough to read image blobs from the host - crioStorageVolumeName = "crio-storage" - crioConfigVolumeName = "crio-config" - // Source: https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md + crioStorageVolumeName = "crio-storage" + crioStorageConfigVolumeName = "crio-storage-config" + crioConfigVolumeName = "crio-config" - crioStorageDefaultPath = "/var/lib/containers/storage" - crioConfigDefaultPath = "/etc/containers/storage.conf" + // Source: https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md and https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md + + crioStorageDefaultPath = "/var/lib/containers" + crioStorageConfigDefaultPath = "/etc/containers/storage.conf" + crioConfigDefaultPath = "/etc/crio/crio.conf" cndrCompanyCodeVarName = "CB_COMPANY_CODES" cndrCompanyCodeKeyName = "companyCode" @@ -213,8 +216,8 @@ func (obj *SensorDaemonSetK8sObject) getExpectedVolumeCount(agentSpec *cbContain } // cri-o specific mounts to use the containers/storage library with the host's storage area - // one mount is for the image store on the host and one is for the storage.config => we expect 2 more - expectedVolumesCount += 2 + // one mount is for the image store on the host, one is for the storage.conf and one is for crio.conf => we expect 3 more + expectedVolumesCount += 3 } if isCndrEnbaled(agentSpec.Components.Cndr) { @@ -489,7 +492,6 @@ func (obj *SensorDaemonSetK8sObject) mutateCndrVolumesMounts(container *coreV1.C } func (obj *SensorDaemonSetK8sObject) mutateClusterScannerEnvVars(container *coreV1.Container, agentSpec *cbContainersV1.CBContainersAgentSpec) { - clusterScannerSpec := &agentSpec.Components.ClusterScanning.ClusterScannerAgent customEnvs := []coreV1.EnvVar{ @@ -555,6 +557,16 @@ func (obj *SensorDaemonSetK8sObject) mutateClusterScannerVolumes(templatePodSpec } templatePodSpec.Volumes[crioStorageIx].HostPath.Path = storagePath + crioStorageConfigIx := commonState.EnsureAndGetVolumeIndexForName(templatePodSpec, crioStorageConfigVolumeName) + storageConfigPath := clusterScannerSpec.K8sContainerEngine.CRIO.StorageConfigPath + if storageConfigPath == "" { + storageConfigPath = crioStorageConfigDefaultPath + } + if templatePodSpec.Volumes[crioStorageConfigIx].HostPath == nil { + templatePodSpec.Volumes[crioStorageConfigIx].HostPath = &coreV1.HostPathVolumeSource{} + } + templatePodSpec.Volumes[crioStorageConfigIx].HostPath.Path = storageConfigPath + crioConfigIx := commonState.EnsureAndGetVolumeIndexForName(templatePodSpec, crioConfigVolumeName) configPath := clusterScannerSpec.K8sContainerEngine.CRIO.ConfigPath if configPath == "" { @@ -564,6 +576,7 @@ func (obj *SensorDaemonSetK8sObject) mutateClusterScannerVolumes(templatePodSpec templatePodSpec.Volumes[crioConfigIx].HostPath = &coreV1.HostPathVolumeSource{} } templatePodSpec.Volumes[crioConfigIx].HostPath.Path = configPath + // End CRI-O config // mutate root-cas volume, for https certificates commonState.MutateVolumesToIncludeRootCAsVolume(templatePodSpec) @@ -573,8 +586,8 @@ func (obj *SensorDaemonSetK8sObject) mutateClusterScannerVolumesMounts(container containerRuntimes := getClusterScannerContainerRuntimes(&agentSpec.Components.ClusterScanning.ClusterScannerAgent) // We expect to see - // container runtimes mounts + root CA mount + 2 mounts for CRI-O - if container.VolumeMounts == nil || len(container.VolumeMounts) != (len(containerRuntimes)+1+2) { + // container runtimes mounts + root CA mount + 3 mounts for CRI-O + if container.VolumeMounts == nil || len(container.VolumeMounts) != (len(containerRuntimes)+1+3) { container.VolumeMounts = make([]coreV1.VolumeMount, 0) } @@ -591,6 +604,8 @@ func (obj *SensorDaemonSetK8sObject) mutateClusterScannerVolumesMounts(container crioStorageIx := commonState.EnsureAndGetVolumeMountIndexForName(container, crioStorageVolumeName) // The storage MUST be R/W as file-based locks are used to enable concurrent access to the same storage from both CRI-O and our agent commonState.MutateVolumeMount(container, crioStorageIx, crioStorageDefaultPath, false) + crioStorageConfigIx := commonState.EnsureAndGetVolumeMountIndexForName(container, crioStorageConfigVolumeName) + commonState.MutateVolumeMount(container, crioStorageConfigIx, crioStorageConfigDefaultPath, true) crioConfigIx := commonState.EnsureAndGetVolumeMountIndexForName(container, crioConfigVolumeName) commonState.MutateVolumeMount(container, crioConfigIx, crioConfigDefaultPath, true) } diff --git a/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml b/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml index 84fdcd10..1009057e 100644 --- a/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml +++ b/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml @@ -3353,8 +3353,18 @@ spec: properties: configPath: description: ConfigPath can be used to set the - path to CRI-O's configuration file If not specified, - the default location for CRI-O is used (/etc/containers/storage.conf). + path to CRI-O's configuration file. If not specified, + the default location is used (/etc/crio/crio.conf). + See https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md + for more information. + type: string + storageConfigPath: + description: StorageConfigPath can be used to + set the path to the storage configuration file + used by CRI-O (if any). If not specified, the + default location for storage is used (/etc/containers/storage.conf). + The files does not need to exist. See https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md + for more information type: string storagePath: description: StoragePath can be used to set the diff --git a/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml b/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml index db04bc93..233baf9c 100644 --- a/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml +++ b/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml @@ -3146,8 +3146,18 @@ spec: properties: configPath: description: ConfigPath can be used to set the path - to CRI-O's configuration file If not specified, - the default location for CRI-O is used (/etc/containers/storage.conf). + to CRI-O's configuration file. If not specified, + the default location is used (/etc/crio/crio.conf). + See https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md + for more information. + type: string + storageConfigPath: + description: StorageConfigPath can be used to set + the path to the storage configuration file used + by CRI-O (if any). If not specified, the default + location for storage is used (/etc/containers/storage.conf). + The files does not need to exist. See https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md + for more information type: string storagePath: description: StoragePath can be used to set the From 73f4dd33604695e388fc4c8a18df3bd7a45d9867 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Tue, 6 Jun 2023 13:19:24 +0300 Subject: [PATCH 9/9] Update helm chart, too --- .../templates/operator.yaml | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml b/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml index 635220c6..98f26019 100644 --- a/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml +++ b/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml @@ -1845,12 +1845,44 @@ spec: k8sContainerEngine: default: {} properties: + CRIO: + default: { } + description: CRIO holds configuration values specific + to the CRI-O container engine + properties: + configPath: + description: ConfigPath can be used to set the + path to CRI-O's configuration file. If not specified, + the default location is used (/etc/crio/crio.conf). + See https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md + for more information. + type: string + storageConfigPath: + description: StorageConfigPath can be used to + set the path to the storage configuration file + used by CRI-O (if any). If not specified, the + default location for storage is used (/etc/containers/storage.conf). + The files does not need to exist. See https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md + for more information + type: string + storagePath: + description: StoragePath can be used to set the + path used by CRI-O to store images on each node. + This path will be mounted on the cluster scanner + to provide access to the node's images. If the + path does not match what CRI-O uses on the nodes, + then images will not be found and scanned as + expected. If not specified, the default location + of CRI-O is used (/var/lib/containers/storage). + type: string + type: object endpoint: type: string engineType: enum: - containerd - docker-daemon + - cri-o type: string type: object labels: