Skip to content

Commit

Permalink
Merge pull request #158 from octarinesec/CNS-1284-crio-cherrypick
Browse files Browse the repository at this point in the history 
Cherry-pick CRI-O support from main into 5.6.2
  • Loading branch information
@yonatan-shorani
yonatan-shorani authored Jun 18, 2023
2 parents 9fd9747 + e808267 commit 4e29baa
Show file tree
Hide file tree
Showing 10 changed files with 240 additions and 33 deletions.
30 changes: 28 additions & 2 deletions api/v1/container_runtime_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,36 @@ const (
CRIOContainerRuntime ContainerEngineType = "cri-o"
)

var SupportedK8sEngineTypes = []ContainerEngineType{ContainerdContainerRuntime, DockerContainerRuntime}
var SupportedK8sEngineTypes = []ContainerEngineType{ContainerdContainerRuntime, DockerContainerRuntime, CRIOContainerRuntime}

type K8sContainerEngineSpec struct {
Endpoint string `json:"endpoint,omitempty"`
// +kubebuilder:validation:Enum:=containerd;docker-daemon
// +kubebuilder:validation:Enum:=containerd;docker-daemon;cri-o
EngineType ContainerEngineType `json:"engineType,omitempty"`

// +kubebuilder:default:=<>
// CRIO holds configuration values specific to the CRI-O container engine
CRIO CRIOSpec `json:"CRIO,omitempty"`
}

type CRIOSpec struct {
// StoragePath can be used to set the path used by CRI-O to store images on each node.
// This path will be mounted on the cluster scanner to provide access to the node's images.
// If the path does not match what CRI-O uses on the nodes, then images will not be found and scanned as expected.
// If not specified, the default location of CRI-O is used (/var/lib/containers/storage).
// +kubebuilder:validation:Optional
StoragePath string `json:"storagePath,omitempty"`

// StorageConfigPath can be used to set the path to the storage configuration file used by CRI-O (if any).
// If not specified, the default location for storage is used (/etc/containers/storage.conf).
// The files does not need to exist.
// See https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md for more information
// +kubebuilder:validation:Optional
StorageConfigPath string `json:"storageConfigPath,omitempty"`

// ConfigPath can be used to set the path to CRI-O's configuration file.
// If not specified, the default location is used (/etc/crio/crio.conf).
// See https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md for more information.
// +kubebuilder:validation:Optional
ConfigPath string `json:"configPath,omitempty"`
}
16 changes: 16 additions & 0 deletions api/v1/zz_generated.deepcopy.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

78 changes: 71 additions & 7 deletions cbcontainers/state/components/sensor_daemon_set.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,16 +28,30 @@ const (

desiredConnectionTimeoutSeconds = 60

// k8s container runtime/contaienr engine endpoints
// k8s container runtime/container engine endpoints
containerdRuntimeEndpoint = "/var/run/containerd/containerd.sock"
microk8sContainerdRuntimeEndpoint = "/var/snap/microk8s/common/run/containerd.sock"
k3sContainerdRuntimeEndpoint = "/run/k3s/containerd/containerd.sock"
dockerRuntimeEndpoint = "/var/run/dockershim.sock"
dockerSock = "/var/run/docker.sock"
crioRuntimeEndpoint = "/var/run/crio/crio.sock"

// configuredContainerRuntimeVolumeName is used when the customer has specified a non-standard runtime endpoint in the CRD
// as this means we need a special volume+mount for this endpoint
configuredContainerRuntimeVolumeName = "configured-container-runtime-endpoint"

// CRI-O specific volumes since the socket is not enough to read image blobs from the host

crioStorageVolumeName = "crio-storage"
crioStorageConfigVolumeName = "crio-storage-config"
crioConfigVolumeName = "crio-config"

// Source: https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md and https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md

crioStorageDefaultPath = "/var/lib/containers"
crioStorageConfigDefaultPath = "/etc/containers/storage.conf"
crioConfigDefaultPath = "/etc/crio/crio.conf"

cndrCompanyCodeVarName = "CB_COMPANY_CODES"
cndrCompanyCodeKeyName = "companyCode"
)
Expand Down Expand Up @@ -192,8 +206,15 @@ func (obj *SensorDaemonSetK8sObject) getExpectedVolumeCount(agentSpec *cbContain

if commonState.IsEnabled(agentSpec.Components.ClusterScanning.Enabled) {
clusterScannerSpec := &agentSpec.Components.ClusterScanning.ClusterScannerAgent
containerRuntimes := getContainerRuntimes(clusterScannerSpec)
expectedVolumesCount += len(containerRuntimes) + 1
expectedVolumesCount += len(supportedContainerRuntimes)
expectedVolumesCount += 1 // RootCA
if clusterScannerSpec.K8sContainerEngine.Endpoint != "" {
expectedVolumesCount += 1
}

// cri-o specific mounts to use the containers/storage library with the host's storage area
// one mount is for the image store on the host, one is for the storage.conf and one is for crio.conf => we expect 3 more
expectedVolumesCount += 3
}

if isCndrEnbaled(agentSpec.Components.Cndr) {
Expand All @@ -206,9 +227,10 @@ func (obj *SensorDaemonSetK8sObject) getExpectedVolumeCount(agentSpec *cbContain
func (obj *SensorDaemonSetK8sObject) mutateVolumes(daemonSet *appsV1.DaemonSet, agentSpec *cbContainersV1.CBContainersAgentSpec) {
templatePodSpec := &daemonSet.Spec.Template.Spec

if templatePodSpec.Volumes == nil || len(templatePodSpec.Volumes) != obj.getExpectedVolumeCount(agentSpec) {
expectedVolumeCount := obj.getExpectedVolumeCount(agentSpec)
if templatePodSpec.Volumes == nil || len(templatePodSpec.Volumes) != expectedVolumeCount {
// clean cluster-scanner & cndr volumes
templatePodSpec.Volumes = make([]coreV1.Volume, 0)
templatePodSpec.Volumes = make([]coreV1.Volume, 0, expectedVolumeCount)
}

if commonState.IsEnabled(agentSpec.Components.ClusterScanning.Enabled) {
Expand Down Expand Up @@ -454,7 +476,6 @@ func (obj *SensorDaemonSetK8sObject) mutateCndrVolumesMounts(container *coreV1.C
}

func (obj *SensorDaemonSetK8sObject) mutateClusterScannerEnvVars(container *coreV1.Container, agentSpec *cbContainersV1.CBContainersAgentSpec) {

clusterScannerSpec := &agentSpec.Components.ClusterScanning.ClusterScannerAgent

customEnvs := []coreV1.EnvVar{
Expand Down Expand Up @@ -502,12 +523,46 @@ func (obj *SensorDaemonSetK8sObject) mutateClusterScannerVolumes(templatePodSpec
}
templatePodSpec.Volumes[routeIndex].HostPath.Path = path
}

// Ensure we have volumes for CRI-O
crioStorageIx := commonState.EnsureAndGetVolumeIndexForName(templatePodSpec, crioStorageVolumeName)
storagePath := clusterScannerSpec.K8sContainerEngine.CRIO.StoragePath
if storagePath == "" {
storagePath = crioStorageDefaultPath
}
if templatePodSpec.Volumes[crioStorageIx].HostPath == nil {
templatePodSpec.Volumes[crioStorageIx].HostPath = &coreV1.HostPathVolumeSource{}
}
templatePodSpec.Volumes[crioStorageIx].HostPath.Path = storagePath

crioStorageConfigIx := commonState.EnsureAndGetVolumeIndexForName(templatePodSpec, crioStorageConfigVolumeName)
storageConfigPath := clusterScannerSpec.K8sContainerEngine.CRIO.StorageConfigPath
if storageConfigPath == "" {
storageConfigPath = crioStorageConfigDefaultPath
}
if templatePodSpec.Volumes[crioStorageConfigIx].HostPath == nil {
templatePodSpec.Volumes[crioStorageConfigIx].HostPath = &coreV1.HostPathVolumeSource{}
}
templatePodSpec.Volumes[crioStorageConfigIx].HostPath.Path = storageConfigPath

crioConfigIx := commonState.EnsureAndGetVolumeIndexForName(templatePodSpec, crioConfigVolumeName)
configPath := clusterScannerSpec.K8sContainerEngine.CRIO.ConfigPath
if configPath == "" {
configPath = crioConfigDefaultPath
}
if templatePodSpec.Volumes[crioConfigIx].HostPath == nil {
templatePodSpec.Volumes[crioConfigIx].HostPath = &coreV1.HostPathVolumeSource{}
}
templatePodSpec.Volumes[crioConfigIx].HostPath.Path = configPath
// End CRI-O config
}

func (obj *SensorDaemonSetK8sObject) mutateClusterScannerVolumesMounts(container *coreV1.Container, agentSpec *cbContainersV1.CBContainersAgentSpec) {
containerRuntimes := getContainerRuntimes(&agentSpec.Components.ClusterScanning.ClusterScannerAgent)

if container.VolumeMounts == nil || len(container.VolumeMounts) != len(containerRuntimes)+1 {
// We expect to see
// container runtimes mounts + root CA mount + 3 mounts for CRI-O
if container.VolumeMounts == nil || len(container.VolumeMounts) != (len(containerRuntimes)+1+3) {
container.VolumeMounts = make([]coreV1.VolumeMount, 0)
}

Expand All @@ -519,6 +574,15 @@ func (obj *SensorDaemonSetK8sObject) mutateClusterScannerVolumesMounts(container
index := commonState.EnsureAndGetVolumeMountIndexForName(container, name)
commonState.MutateVolumeMount(container, index, mountPath, true)
}

// mutate mounts for the CRI-O engine storage
crioStorageIx := commonState.EnsureAndGetVolumeMountIndexForName(container, crioStorageVolumeName)
// The storage MUST be R/W as file-based locks are used to enable concurrent access to the same storage from both CRI-O and our agent
commonState.MutateVolumeMount(container, crioStorageIx, crioStorageDefaultPath, false)
crioStorageConfigIx := commonState.EnsureAndGetVolumeMountIndexForName(container, crioStorageConfigVolumeName)
commonState.MutateVolumeMount(container, crioStorageConfigIx, crioStorageConfigDefaultPath, true)
crioConfigIx := commonState.EnsureAndGetVolumeMountIndexForName(container, crioConfigVolumeName)
commonState.MutateVolumeMount(container, crioConfigIx, crioConfigDefaultPath, true)
}

// The cluster scanner uses the container runtime unix socket to fetch the running containers to scan.
Expand Down
32 changes: 32 additions & 0 deletions charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1845,12 +1845,44 @@ spec:
k8sContainerEngine:
default: {}
properties:
CRIO:
default: { }
description: CRIO holds configuration values specific
to the CRI-O container engine
properties:
configPath:
description: ConfigPath can be used to set the
path to CRI-O's configuration file. If not specified,
the default location is used (/etc/crio/crio.conf).
See https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md
for more information.
type: string
storageConfigPath:
description: StorageConfigPath can be used to
set the path to the storage configuration file
used by CRI-O (if any). If not specified, the
default location for storage is used (/etc/containers/storage.conf).
The files does not need to exist. See https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md
for more information
type: string
storagePath:
description: StoragePath can be used to set the
path used by CRI-O to store images on each node.
This path will be mounted on the cluster scanner
to provide access to the node's images. If the
path does not match what CRI-O uses on the nodes,
then images will not be found and scanned as
expected. If not specified, the default location
of CRI-O is used (/var/lib/containers/storage).
type: string
type: object
endpoint:
type: string
engineType:
enum:
- containerd
- docker-daemon
- cri-o
type: string
type: object
labels:
Expand Down
32 changes: 32 additions & 0 deletions config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3346,12 +3346,44 @@ spec:
k8sContainerEngine:
default: {}
properties:
CRIO:
default: {}
description: CRIO holds configuration values specific
to the CRI-O container engine
properties:
configPath:
description: ConfigPath can be used to set the
path to CRI-O's configuration file. If not specified,
the default location is used (/etc/crio/crio.conf).
See https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md
for more information.
type: string
storageConfigPath:
description: StorageConfigPath can be used to
set the path to the storage configuration file
used by CRI-O (if any). If not specified, the
default location for storage is used (/etc/containers/storage.conf).
The files does not need to exist. See https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md
for more information
type: string
storagePath:
description: StoragePath can be used to set the
path used by CRI-O to store images on each node.
This path will be mounted on the cluster scanner
to provide access to the node's images. If the
path does not match what CRI-O uses on the nodes,
then images will not be found and scanned as
expected. If not specified, the default location
of CRI-O is used (/var/lib/containers/storage).
type: string
type: object
endpoint:
type: string
engineType:
enum:
- containerd
- docker-daemon
- cri-o
type: string
type: object
labels:
Expand Down
31 changes: 31 additions & 0 deletions config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3140,12 +3140,43 @@ spec:
type: object
k8sContainerEngine:
properties:
CRIO:
description: CRIO holds configuration values specific
to the CRI-O container engine
properties:
configPath:
description: ConfigPath can be used to set the path
to CRI-O's configuration file. If not specified,
the default location is used (/etc/crio/crio.conf).
See https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md
for more information.
type: string
storageConfigPath:
description: StorageConfigPath can be used to set
the path to the storage configuration file used
by CRI-O (if any). If not specified, the default
location for storage is used (/etc/containers/storage.conf).
The files does not need to exist. See https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md
for more information
type: string
storagePath:
description: StoragePath can be used to set the
path used by CRI-O to store images on each node.
This path will be mounted on the cluster scanner
to provide access to the node's images. If the
path does not match what CRI-O uses on the nodes,
then images will not be found and scanned as expected.
If not specified, the default location of CRI-O
is used (/var/lib/containers/storage).
type: string
type: object
endpoint:
type: string
engineType:
enum:
- containerd
- docker-daemon
- cri-o
type: string
type: object
labels:
Expand Down
2 changes: 1 addition & 1 deletion controllers/cbcontainersagent_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ func (r *CBContainersAgentController) Reconcile(ctx context.Context, req ctrl.Re
}

if err := r.setAgentDefaults(&cbContainersAgent.Spec); err != nil {
return ctrl.Result{}, fmt.Errorf("faild to set defaults to cluster CR: %v", err)
return ctrl.Result{}, fmt.Errorf("failed to set defaults to cluster CR: %v", err)
}

setOwner := func(controlledResource metav1.Object) error {
Expand Down
Loading

0 comments on commit 4e29baa

Please sign in to comment.