Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cherry-pick CRI-O support from main into 5.6.2 #158

Merged
merged 1 commit into from
Jun 18, 2023
Merged

Cherry-pick CRI-O support from main into 5.6.2 #158

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 28 additions & 2 deletions api/v1/container_runtime_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,36 @@ const (
CRIOContainerRuntime ContainerEngineType = "cri-o"
)

var SupportedK8sEngineTypes = []ContainerEngineType{ContainerdContainerRuntime, DockerContainerRuntime}
var SupportedK8sEngineTypes = []ContainerEngineType{ContainerdContainerRuntime, DockerContainerRuntime, CRIOContainerRuntime}

type K8sContainerEngineSpec struct {
Endpoint string `json:"endpoint,omitempty"`
// +kubebuilder:validation:Enum:=containerd;docker-daemon
// +kubebuilder:validation:Enum:=containerd;docker-daemon;cri-o
EngineType ContainerEngineType `json:"engineType,omitempty"`

// +kubebuilder:default:=<>
// CRIO holds configuration values specific to the CRI-O container engine
CRIO CRIOSpec `json:"CRIO,omitempty"`
}

type CRIOSpec struct {
// StoragePath can be used to set the path used by CRI-O to store images on each node.
// This path will be mounted on the cluster scanner to provide access to the node's images.
// If the path does not match what CRI-O uses on the nodes, then images will not be found and scanned as expected.
// If not specified, the default location of CRI-O is used (/var/lib/containers/storage).
// +kubebuilder:validation:Optional
StoragePath string `json:"storagePath,omitempty"`

// StorageConfigPath can be used to set the path to the storage configuration file used by CRI-O (if any).
// If not specified, the default location for storage is used (/etc/containers/storage.conf).
// The files does not need to exist.
// See https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md for more information
// +kubebuilder:validation:Optional
StorageConfigPath string `json:"storageConfigPath,omitempty"`

// ConfigPath can be used to set the path to CRI-O's configuration file.
// If not specified, the default location is used (/etc/crio/crio.conf).
// See https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md for more information.
// +kubebuilder:validation:Optional
ConfigPath string `json:"configPath,omitempty"`
}
16 changes: 16 additions & 0 deletions api/v1/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

78 changes: 71 additions & 7 deletions cbcontainers/state/components/sensor_daemon_set.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,16 +28,30 @@ const (

desiredConnectionTimeoutSeconds = 60

// k8s container runtime/contaienr engine endpoints
// k8s container runtime/container engine endpoints
containerdRuntimeEndpoint = "/var/run/containerd/containerd.sock"
microk8sContainerdRuntimeEndpoint = "/var/snap/microk8s/common/run/containerd.sock"
k3sContainerdRuntimeEndpoint = "/run/k3s/containerd/containerd.sock"
dockerRuntimeEndpoint = "/var/run/dockershim.sock"
dockerSock = "/var/run/docker.sock"
crioRuntimeEndpoint = "/var/run/crio/crio.sock"

// configuredContainerRuntimeVolumeName is used when the customer has specified a non-standard runtime endpoint in the CRD
// as this means we need a special volume+mount for this endpoint
configuredContainerRuntimeVolumeName = "configured-container-runtime-endpoint"

// CRI-O specific volumes since the socket is not enough to read image blobs from the host

crioStorageVolumeName = "crio-storage"
crioStorageConfigVolumeName = "crio-storage-config"
crioConfigVolumeName = "crio-config"

// Source: https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md and https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md

crioStorageDefaultPath = "/var/lib/containers"
crioStorageConfigDefaultPath = "/etc/containers/storage.conf"
crioConfigDefaultPath = "/etc/crio/crio.conf"

cndrCompanyCodeVarName = "CB_COMPANY_CODES"
cndrCompanyCodeKeyName = "companyCode"
)
Expand Down Expand Up @@ -192,8 +206,15 @@ func (obj *SensorDaemonSetK8sObject) getExpectedVolumeCount(agentSpec *cbContain

if commonState.IsEnabled(agentSpec.Components.ClusterScanning.Enabled) {
clusterScannerSpec := &agentSpec.Components.ClusterScanning.ClusterScannerAgent
containerRuntimes := getContainerRuntimes(clusterScannerSpec)
expectedVolumesCount += len(containerRuntimes) + 1
expectedVolumesCount += len(supportedContainerRuntimes)
expectedVolumesCount += 1 // RootCA
if clusterScannerSpec.K8sContainerEngine.Endpoint != "" {
expectedVolumesCount += 1
}

// cri-o specific mounts to use the containers/storage library with the host's storage area
// one mount is for the image store on the host, one is for the storage.conf and one is for crio.conf => we expect 3 more
expectedVolumesCount += 3
}

if isCndrEnbaled(agentSpec.Components.Cndr) {
Expand All @@ -206,9 +227,10 @@ func (obj *SensorDaemonSetK8sObject) getExpectedVolumeCount(agentSpec *cbContain
func (obj *SensorDaemonSetK8sObject) mutateVolumes(daemonSet *appsV1.DaemonSet, agentSpec *cbContainersV1.CBContainersAgentSpec) {
templatePodSpec := &daemonSet.Spec.Template.Spec

if templatePodSpec.Volumes == nil || len(templatePodSpec.Volumes) != obj.getExpectedVolumeCount(agentSpec) {
expectedVolumeCount := obj.getExpectedVolumeCount(agentSpec)
if templatePodSpec.Volumes == nil || len(templatePodSpec.Volumes) != expectedVolumeCount {
// clean cluster-scanner & cndr volumes
templatePodSpec.Volumes = make([]coreV1.Volume, 0)
templatePodSpec.Volumes = make([]coreV1.Volume, 0, expectedVolumeCount)
}

if commonState.IsEnabled(agentSpec.Components.ClusterScanning.Enabled) {
Expand Down Expand Up @@ -454,7 +476,6 @@ func (obj *SensorDaemonSetK8sObject) mutateCndrVolumesMounts(container *coreV1.C
}

func (obj *SensorDaemonSetK8sObject) mutateClusterScannerEnvVars(container *coreV1.Container, agentSpec *cbContainersV1.CBContainersAgentSpec) {

clusterScannerSpec := &agentSpec.Components.ClusterScanning.ClusterScannerAgent

customEnvs := []coreV1.EnvVar{
Expand Down Expand Up @@ -502,12 +523,46 @@ func (obj *SensorDaemonSetK8sObject) mutateClusterScannerVolumes(templatePodSpec
}
templatePodSpec.Volumes[routeIndex].HostPath.Path = path
}

// Ensure we have volumes for CRI-O
crioStorageIx := commonState.EnsureAndGetVolumeIndexForName(templatePodSpec, crioStorageVolumeName)
storagePath := clusterScannerSpec.K8sContainerEngine.CRIO.StoragePath
if storagePath == "" {
storagePath = crioStorageDefaultPath
}
if templatePodSpec.Volumes[crioStorageIx].HostPath == nil {
templatePodSpec.Volumes[crioStorageIx].HostPath = &coreV1.HostPathVolumeSource{}
}
templatePodSpec.Volumes[crioStorageIx].HostPath.Path = storagePath

crioStorageConfigIx := commonState.EnsureAndGetVolumeIndexForName(templatePodSpec, crioStorageConfigVolumeName)
storageConfigPath := clusterScannerSpec.K8sContainerEngine.CRIO.StorageConfigPath
if storageConfigPath == "" {
storageConfigPath = crioStorageConfigDefaultPath
}
if templatePodSpec.Volumes[crioStorageConfigIx].HostPath == nil {
templatePodSpec.Volumes[crioStorageConfigIx].HostPath = &coreV1.HostPathVolumeSource{}
}
templatePodSpec.Volumes[crioStorageConfigIx].HostPath.Path = storageConfigPath

crioConfigIx := commonState.EnsureAndGetVolumeIndexForName(templatePodSpec, crioConfigVolumeName)
configPath := clusterScannerSpec.K8sContainerEngine.CRIO.ConfigPath
if configPath == "" {
configPath = crioConfigDefaultPath
}
if templatePodSpec.Volumes[crioConfigIx].HostPath == nil {
templatePodSpec.Volumes[crioConfigIx].HostPath = &coreV1.HostPathVolumeSource{}
}
templatePodSpec.Volumes[crioConfigIx].HostPath.Path = configPath
// End CRI-O config
}

func (obj *SensorDaemonSetK8sObject) mutateClusterScannerVolumesMounts(container *coreV1.Container, agentSpec *cbContainersV1.CBContainersAgentSpec) {
containerRuntimes := getContainerRuntimes(&agentSpec.Components.ClusterScanning.ClusterScannerAgent)

if container.VolumeMounts == nil || len(container.VolumeMounts) != len(containerRuntimes)+1 {
// We expect to see
// container runtimes mounts + root CA mount + 3 mounts for CRI-O
if container.VolumeMounts == nil || len(container.VolumeMounts) != (len(containerRuntimes)+1+3) {
container.VolumeMounts = make([]coreV1.VolumeMount, 0)
}

Expand All @@ -519,6 +574,15 @@ func (obj *SensorDaemonSetK8sObject) mutateClusterScannerVolumesMounts(container
index := commonState.EnsureAndGetVolumeMountIndexForName(container, name)
commonState.MutateVolumeMount(container, index, mountPath, true)
}

// mutate mounts for the CRI-O engine storage
crioStorageIx := commonState.EnsureAndGetVolumeMountIndexForName(container, crioStorageVolumeName)
// The storage MUST be R/W as file-based locks are used to enable concurrent access to the same storage from both CRI-O and our agent
commonState.MutateVolumeMount(container, crioStorageIx, crioStorageDefaultPath, false)
crioStorageConfigIx := commonState.EnsureAndGetVolumeMountIndexForName(container, crioStorageConfigVolumeName)
commonState.MutateVolumeMount(container, crioStorageConfigIx, crioStorageConfigDefaultPath, true)
crioConfigIx := commonState.EnsureAndGetVolumeMountIndexForName(container, crioConfigVolumeName)
commonState.MutateVolumeMount(container, crioConfigIx, crioConfigDefaultPath, true)
}

// The cluster scanner uses the container runtime unix socket to fetch the running containers to scan.
Expand Down
32 changes: 32 additions & 0 deletions charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1845,12 +1845,44 @@ spec:
k8sContainerEngine:
default: {}
properties:
CRIO:
default: { }
description: CRIO holds configuration values specific
to the CRI-O container engine
properties:
configPath:
description: ConfigPath can be used to set the
path to CRI-O's configuration file. If not specified,
the default location is used (/etc/crio/crio.conf).
See https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md
for more information.
type: string
storageConfigPath:
description: StorageConfigPath can be used to
set the path to the storage configuration file
used by CRI-O (if any). If not specified, the
default location for storage is used (/etc/containers/storage.conf).
The files does not need to exist. See https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md
for more information
type: string
storagePath:
description: StoragePath can be used to set the
path used by CRI-O to store images on each node.
This path will be mounted on the cluster scanner
to provide access to the node's images. If the
path does not match what CRI-O uses on the nodes,
then images will not be found and scanned as
expected. If not specified, the default location
of CRI-O is used (/var/lib/containers/storage).
type: string
type: object
endpoint:
type: string
engineType:
enum:
- containerd
- docker-daemon
- cri-o
type: string
type: object
labels:
Expand Down
32 changes: 32 additions & 0 deletions config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3346,12 +3346,44 @@ spec:
k8sContainerEngine:
default: {}
properties:
CRIO:
default: {}
description: CRIO holds configuration values specific
to the CRI-O container engine
properties:
configPath:
description: ConfigPath can be used to set the
path to CRI-O's configuration file. If not specified,
the default location is used (/etc/crio/crio.conf).
See https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md
for more information.
type: string
storageConfigPath:
description: StorageConfigPath can be used to
set the path to the storage configuration file
used by CRI-O (if any). If not specified, the
default location for storage is used (/etc/containers/storage.conf).
The files does not need to exist. See https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md
for more information
type: string
storagePath:
description: StoragePath can be used to set the
path used by CRI-O to store images on each node.
This path will be mounted on the cluster scanner
to provide access to the node's images. If the
path does not match what CRI-O uses on the nodes,
then images will not be found and scanned as
expected. If not specified, the default location
of CRI-O is used (/var/lib/containers/storage).
type: string
type: object
endpoint:
type: string
engineType:
enum:
- containerd
- docker-daemon
- cri-o
type: string
type: object
labels:
Expand Down
31 changes: 31 additions & 0 deletions config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3140,12 +3140,43 @@ spec:
type: object
k8sContainerEngine:
properties:
CRIO:
description: CRIO holds configuration values specific
to the CRI-O container engine
properties:
configPath:
description: ConfigPath can be used to set the path
to CRI-O's configuration file. If not specified,
the default location is used (/etc/crio/crio.conf).
See https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md
for more information.
type: string
storageConfigPath:
description: StorageConfigPath can be used to set
the path to the storage configuration file used
by CRI-O (if any). If not specified, the default
location for storage is used (/etc/containers/storage.conf).
The files does not need to exist. See https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md
for more information
type: string
storagePath:
description: StoragePath can be used to set the
path used by CRI-O to store images on each node.
This path will be mounted on the cluster scanner
to provide access to the node's images. If the
path does not match what CRI-O uses on the nodes,
then images will not be found and scanned as expected.
If not specified, the default location of CRI-O
is used (/var/lib/containers/storage).
type: string
type: object
endpoint:
type: string
engineType:
enum:
- containerd
- docker-daemon
- cri-o
type: string
type: object
labels:
Expand Down
2 changes: 1 addition & 1 deletion controllers/cbcontainersagent_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ func (r *CBContainersAgentController) Reconcile(ctx context.Context, req ctrl.Re
}

if err := r.setAgentDefaults(&cbContainersAgent.Spec); err != nil {
return ctrl.Result{}, fmt.Errorf("faild to set defaults to cluster CR: %v", err)
return ctrl.Result{}, fmt.Errorf("failed to set defaults to cluster CR: %v", err)
}

setOwner := func(controlledResource metav1.Object) error {
Expand Down
Loading